From 4fa0e91bd77290614e721718691bb76ac2dc7f0e Mon Sep 17 00:00:00 2001 From: palaviv Date: Fri, 24 Feb 2017 12:23:19 +0200 Subject: [PATCH 01/77] support BLOB incremental I/O in sqlite module --- Doc/includes/sqlite3/blob.py | 13 ++ Doc/includes/sqlite3/blob_with.py | 12 ++ Doc/library/sqlite3.rst | 67 ++++++ Lib/sqlite3/test/dbapi.py | 279 +++++++++++++++++++++++- Modules/_sqlite/blob.c | 342 ++++++++++++++++++++++++++++++ Modules/_sqlite/blob.h | 25 +++ Modules/_sqlite/connection.c | 93 +++++++- Modules/_sqlite/connection.h | 3 +- Modules/_sqlite/module.c | 4 +- setup.py | 3 +- 10 files changed, 830 insertions(+), 11 deletions(-) create mode 100644 Doc/includes/sqlite3/blob.py create mode 100644 Doc/includes/sqlite3/blob_with.py create mode 100644 Modules/_sqlite/blob.c create mode 100644 Modules/_sqlite/blob.h diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py new file mode 100644 index 00000000000000..b8d2c78d1a365d --- /dev/null +++ b/Doc/includes/sqlite3/blob.py @@ -0,0 +1,13 @@ +import sqlite3 + +con = sqlite3.connect(":memory:") +# creating the table +con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("insert into test(blob_col) values (zeroblob(10))") +# opening blob handle +blob = con.open_blob("test", "blob_col", 1, 1) +blob.write(b"a" * 5) +blob.write(b"b" * 5) +blob.seek(0) +print(blob.read()) # will print b"aaaaabbbbb" +blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py new file mode 100644 index 00000000000000..624b680591901a --- /dev/null +++ b/Doc/includes/sqlite3/blob_with.py @@ -0,0 +1,12 @@ +import sqlite3 + +con = sqlite3.connect(":memory:") +# creating the table +con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("insert into test(blob_col) values (zeroblob(10))") +# opening blob handle +with con.open_blob("test", "blob_col", 1, 1) as blob: + blob.write(b"a" * 5) + blob.write(b"b" * 5) + blob.seek(0) + print(blob.read()) # will print b"aaaaabbbbb" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ccb82278bdaa13..ceb92b3fe90fab 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -301,6 +301,16 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. + .. method:: open_blob(table, column, row, readonly=False, dbname="main") + + On success a :class:`Blob` handle to the + :abbr:`BLOB (Binary Large OBject)` located in row *row*, + column *column*, table *table* in database *dbname* will be returned. + When *readonly* is :const:`True` the BLOB is opened with read + permissions. Otherwise the BLOB has read and write permissions. + + .. versionadded:: 3.7 + .. method:: commit() This method commits the current transaction. If you don't call this method, @@ -853,6 +863,63 @@ Exceptions transactions turned off. It is a subclass of :exc:`DatabaseError`. +.. _sqlite3-blob-objects: + +Blob Objects +------------ + +.. versionadded:: 3.7 + +.. class:: Blob + + A :class:`Blob` instance can read and write the data in the + :abbr:`BLOB (Binary Large OBject)`. + + .. method:: Blob.close() + + Close the BLOB now (rather than whenever __del__ is called). + + The BLOB will be unusable from this point forward; an + :class:`~sqlite3.Error` (or subclass) exception will be + raised if any operation is attempted with the BLOB. + + .. method:: Blob.__len__() + + Return the BLOB size. + + .. method:: Blob.read([size]) + + Read *size* bytes of data from the BLOB at the current offset position. + If the end of the BLOB is reached we will return the data up to end of + file. When *size* is not specified or negative we will read up to end + of BLOB. + + .. method:: Blob.write(data) + + Write *data* to the BLOB at the current offset. This function cannot + changed BLOB length. If data write will result in writing to more + then BLOB current size an error will be raised. + + .. method:: Blob.tell() + + Return the current offset of the BLOB. + + .. method:: Blob.seek(offset, [whence]) + + Set the BLOB offset. The *whence* argument is optional and defaults to + :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values + are :data:`os.SEEK_CUR` or 1 (seek relative to the current position) and + :data:`os.SEEK_END` or 2 (seek relative to the BLOB’s end). + + :class:`Blob` example: + + .. literalinclude:: ../includes/sqlite3/blob.py + + A :class:`Blob` can also be used with :term:`context manager`: + + .. literalinclude:: ../includes/sqlite3/blob_with.py + + .. _sqlite3-types: SQLite and Python types diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index be11337154bdd2..85db83ee51f246 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -511,6 +511,167 @@ def CheckLastRowIDInsertOR(self): self.assertEqual(results, expected) +class BlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.blob_data = b"a" * 100 + self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data, )) + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.second_data = b"b" * 100 + + def tearDown(self): + self.blob.close() + self.cx.close() + + def CheckLength(self): + self.assertEqual(len(self.blob), 100) + + def CheckTell(self): + self.assertEqual(self.blob.tell(), 0) + + def CheckSeekFromBlobStart(self): + self.blob.seek(10) + self.assertEqual(self.blob.tell(), 10) + self.blob.seek(10, 0) + self.assertEqual(self.blob.tell(), 10) + + def CheckSeekFromCurrentPosition(self): + self.blob.seek(10, 1) + self.blob.seek(10, 1) + self.assertEqual(self.blob.tell(), 20) + + def CheckSeekFromBlobEnd(self): + self.blob.seek(-10, 2) + self.assertEqual(self.blob.tell(), 90) + + def CheckBlobSeekOverBlobSize(self): + try: + self.blob.seek(1000) + self.fail("should have raised a ValueError") + except ValueError: + pass + except Exception: + self.fail("should have raised a ValueError") + + def CheckBlobSeekUnderBlobSize(self): + try: + self.blob.seek(-10) + self.fail("should have raised a ValueError") + except ValueError: + pass + except Exception: + self.fail("should have raised a ValueError") + + def CheckBlobRead(self): + self.assertEqual(self.blob.read(), self.blob_data) + + def CheckBlobReadSize(self): + self.assertEqual(len(self.blob.read(10)), 10) + + def CheckBlobReadAdvanceOffset(self): + self.blob.read(10) + self.assertEqual(self.blob.tell(), 10) + + def CheckBlobReadStartAtOffset(self): + self.blob.seek(10) + self.blob.write(self.second_data[:10]) + self.blob.seek(10) + self.assertEqual(self.blob.read(10), self.second_data[:10]) + + def CheckBlobWrite(self): + self.blob.write(self.second_data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], self.second_data) + + def CheckBlobWriteAtOffset(self): + self.blob.seek(50) + self.blob.write(self.second_data[:50]) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], + self.blob_data[:50] + self.second_data[:50]) + + def CheckBlobWriteAdvanceOffset(self): + self.blob.write(self.second_data[:50]) + self.assertEqual(self.blob.tell(), 50) + + def CheckBlobWriteMoreThenBlobSize(self): + try: + self.blob.write(b"a" * 1000) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobReadAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + try: + self.blob.read() + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobWriteAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + try: + self.blob.write(b"aaa") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobWriteWhenReadOnly(self): + read_only_blob = \ + self.cx.open_blob("test", "blob_col", 1, readonly=True) + try: + read_only_blob.write(b"aaa") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + read_only_blob.close() + + def CheckBlobOpenWithBadDb(self): + try: + self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadTable(self): + try: + self.cx.open_blob("notexisintg", "blob_col", 1) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadColumn(self): + try: + self.cx.open_blob("test", "notexisting", 1) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadRow(self): + try: + self.cx.open_blob("test", "blob_col", 2) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + +@unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -768,6 +929,20 @@ def CheckClosedCurExecute(self): with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4") + def CheckClosedBlobRead(self): + con = sqlite.connect(":memory:") + con.execute("create table test(id integer primary key, blob_col blob)") + con.execute("insert into test(blob_col) values (zeroblob(100))") + blob = con.open_blob("test", "blob_col", 1) + con.close() + try: + blob.read() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except: + self.fail("Should have raised a ProgrammingError") + def CheckClosedCreateFunction(self): con = sqlite.connect(":memory:") con.close() @@ -921,6 +1096,99 @@ def CheckOnConflictReplace(self): self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')]) + +class ClosedBlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckClosedRead(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.read() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedWrite(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.write(b"aaaaaaaaa") + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedSeek(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.seek(10) + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedTell(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.tell() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedClose(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.close() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + +class BlobContextManagerTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckContextExecute(self): + data = b"a" * 100 + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.write(data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) + + def CheckContextCloseBlob(self): + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.seek(10) + try: + blob.close() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def suite(): module_suite = unittest.makeSuite(ModuleTests, "Check") connection_suite = unittest.makeSuite(ConnectionTests, "Check") @@ -931,11 +1199,12 @@ def suite(): closed_con_suite = unittest.makeSuite(ClosedConTests, "Check") closed_cur_suite = unittest.makeSuite(ClosedCurTests, "Check") on_conflict_suite = unittest.makeSuite(SqliteOnConflictTests, "Check") - return unittest.TestSuite(( - module_suite, connection_suite, cursor_suite, thread_suite, - constructor_suite, ext_suite, closed_con_suite, closed_cur_suite, - on_conflict_suite, - )) + blob_suite = unittest.makeSuite(BlobTests, "Check") + closed_blob_suite = unittest.makeSuite(ClosedBlobTests, "Check") + blob_context_manager_suite = unittest.makeSuite(BlobContextManagerTests, "Check") + return unittest.TestSuite((module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, + ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, + blob_suite, closed_blob_suite, blob_context_manager_suite)) def test(): runner = unittest.TextTestRunner() diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c new file mode 100644 index 00000000000000..11da2b0ad5d691 --- /dev/null +++ b/Modules/_sqlite/blob.c @@ -0,0 +1,342 @@ +#include "blob.h" +#include "util.h" + + +int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, + sqlite3_blob *blob) +{ + Py_INCREF(connection); + self->connection = connection; + self->offset = 0; + self->blob = blob; + self->in_weakreflist = NULL; + + if (!pysqlite_check_thread(self->connection)) { + return -1; + } + return 0; +} + +static void remove_blob_from_connection_blob_list(pysqlite_Blob *self) +{ + Py_ssize_t i; + PyObject *item; + + for (i = 0; i < PyList_GET_SIZE(self->connection->blobs); i++) { + item = PyList_GET_ITEM(self->connection->blobs, i); + if (PyWeakref_GetObject(item) == (PyObject *)self) { + PyList_SetSlice(self->connection->blobs, i, i+1, NULL); + break; + } + } +} + +static void _close_blob_inner(pysqlite_Blob* self) +{ + sqlite3_blob *blob; + + /* close the blob */ + blob = self->blob; + self->blob = NULL; + if (blob) { + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + } + + /* remove from connection weaklist */ + remove_blob_from_connection_blob_list(self); + if (self->in_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject*)self); + } +} + +static void pysqlite_blob_dealloc(pysqlite_Blob* self) +{ + _close_blob_inner(self); + Py_XDECREF(self->connection); + Py_TYPE(self)->tp_free((PyObject*)self); +} + + +/* + * Checks if a blob object is usable (i. e. not closed). + * + * 0 => error; 1 => ok + */ +int pysqlite_check_blob(pysqlite_Blob *blob) +{ + + if (!blob->blob) { + PyErr_SetString(pysqlite_ProgrammingError, + "Cannot operate on a closed blob."); + return 0; + } else if (!pysqlite_check_connection(blob->connection) || + !pysqlite_check_thread(blob->connection)) { + return 0; + } else { + return 1; + } +} + + +PyObject* pysqlite_blob_close(pysqlite_Blob *self) +{ + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + _close_blob_inner(self); + Py_RETURN_NONE; +}; + + +static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) +{ + int blob_length; + if (!pysqlite_check_blob(self)) { + return -1; + } + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + return blob_length; +}; + + +PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) +{ + int read_length = -1; + int blob_length = 0; + PyObject *buffer; + char *raw_buffer; + int rc; + + if (!PyArg_ParseTuple(args, "|i", &read_length)) { + return NULL; + } + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + + /* TODO: make this multithreaded and safe! */ + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (read_length < 0) { + /* same as file read. */ + read_length = blob_length; + } + + /* making sure we don't read more then blob size */ + if (read_length > blob_length - self->offset) { + read_length = blob_length - self->offset; + } + + buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (!buffer) { + return NULL; + } + raw_buffer = PyBytes_AS_STRING(buffer); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + Py_DECREF(buffer); + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return NULL; + } + + /* update offset. */ + self->offset += read_length; + + return buffer; +}; + + +PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) +{ + Py_buffer data_buffer; + int rc; + + if (PyObject_GetBuffer(data, &data_buffer, PyBUF_SIMPLE) < 0) { + return NULL; + } + + if (data_buffer.len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "data longer than INT_MAX bytes"); + PyBuffer_Release(&data_buffer); + return NULL; + } + + if (!pysqlite_check_blob(self)) { + PyBuffer_Release(&data_buffer); + return NULL; + } + + /* TODO: throw better error on data bigger then blob. */ + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buffer.buf, + data_buffer.len, self->offset); + Py_END_ALLOW_THREADS + if (rc != SQLITE_OK) { + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyBuffer_Release(&data_buffer); + return NULL; + } + + self->offset += (int)data_buffer.len; + PyBuffer_Release(&data_buffer); + Py_RETURN_NONE; +} + + +PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) +{ + int blob_length, offset, from_what = 0; + + if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { + return NULL; + } + + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + switch (from_what) { + case 0: // relative to blob begin + break; + case 1: // relative to current position + if (offset > INT_MAX - self->offset) { + goto overflow; + } + offset = self->offset + offset; + break; + case 2: // relative to blob end + if (offset > INT_MAX - blob_length) { + goto overflow; + } + offset = blob_length + offset; + break; + default: + return PyErr_Format(PyExc_ValueError, + "from_what should be 0, 1 or 2"); + } + + if (offset < 0 || offset > blob_length) { + return PyErr_Format(PyExc_ValueError, "offset out of blob range"); + } + + self->offset = offset; + Py_RETURN_NONE; + +overflow: + return PyErr_Format(PyExc_OverflowError, "seek offset result in overflow"); +} + + +PyObject* pysqlite_blob_tell(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + return PyLong_FromLong(self->offset); +} + + +PyObject* pysqlite_blob_enter(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_INCREF(self); + return (PyObject *)self; +} + + +PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) +{ + PyObject *res; + if (!pysqlite_check_blob(self)) { + return NULL; + } + + res = pysqlite_blob_close(self); + if (!res) { + return NULL; + } + Py_XDECREF(res); + + Py_RETURN_FALSE; +} + + +static PyMethodDef blob_methods[] = { + {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, + PyDoc_STR("read data from blob")}, + {"write", (PyCFunction)pysqlite_blob_write, METH_O, + PyDoc_STR("write data to blob")}, + {"close", (PyCFunction)pysqlite_blob_close, METH_NOARGS, + PyDoc_STR("close blob")}, + {"seek", (PyCFunction)pysqlite_blob_seek, METH_VARARGS, + PyDoc_STR("change blob current offset")}, + {"tell", (PyCFunction)pysqlite_blob_tell, METH_NOARGS, + PyDoc_STR("return blob current offset")}, + {"__enter__", (PyCFunction)pysqlite_blob_enter, METH_NOARGS, + PyDoc_STR("blob context manager enter")}, + {"__exit__", (PyCFunction)pysqlite_blob_exit, METH_VARARGS, + PyDoc_STR("blob context manager exit")}, + {NULL, NULL} +}; + +static PySequenceMethods blob_sequence_methods = { + (lenfunc)pysqlite_blob_length, /* sq_length */ +}; + + +PyTypeObject pysqlite_BlobType = { + PyVarObject_HEAD_INIT(NULL, 0) + MODULE_NAME ".Blob", + .tp_basicsize = sizeof(pysqlite_Blob), + .tp_dealloc = (destructor)pysqlite_blob_dealloc, + .tp_as_sequence = &blob_sequence_methods, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), + .tp_methods = blob_methods, +}; + +extern int pysqlite_blob_setup_types(void) +{ + pysqlite_BlobType.tp_new = PyType_GenericNew; + return PyType_Ready(&pysqlite_BlobType); +} diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h new file mode 100644 index 00000000000000..204918fd4f01c6 --- /dev/null +++ b/Modules/_sqlite/blob.h @@ -0,0 +1,25 @@ +#ifndef PYSQLITE_BLOB_H +#define PYSQLITE_BLOB_H +#include "Python.h" +#include "sqlite3.h" +#include "connection.h" + +typedef struct +{ + PyObject_HEAD + pysqlite_Connection* connection; + sqlite3_blob *blob; + int offset; + + PyObject* in_weakreflist; /* List of weak references */ +} pysqlite_Blob; + +extern PyTypeObject pysqlite_BlobType; + +int pysqlite_blob_init(pysqlite_Blob* self, pysqlite_Connection* connection, + sqlite3_blob *blob); +PyObject* pysqlite_blob_close(pysqlite_Blob *self); + +int pysqlite_blob_setup_types(void); + +#endif diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 958be7d869794a..b99fcdb24fc25f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -27,6 +27,7 @@ #include "connection.h" #include "statement.h" #include "cursor.h" +#include "blob.h" #include "prepare_protocol.h" #include "util.h" @@ -105,7 +106,8 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject Py_CLEAR(self->statement_cache); Py_CLEAR(self->statements); Py_CLEAR(self->cursors); - + Py_CLEAR(self->blobs); + Py_INCREF(Py_None); Py_XSETREF(self->row_factory, Py_None); @@ -159,10 +161,11 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject self->created_statements = 0; self->created_cursors = 0; - /* Create lists of weak references to statements/cursors */ + /* Create lists of weak references to statements/cursors/blobs */ self->statements = PyList_New(0); self->cursors = PyList_New(0); - if (!self->statements || !self->cursors) { + self->blobs = PyList_New(0); + if (!self->statements || !self->cursors || !self->blobs) { return -1; } @@ -258,6 +261,8 @@ void pysqlite_connection_dealloc(pysqlite_Connection* self) Py_XDECREF(self->collations); Py_XDECREF(self->statements); Py_XDECREF(self->cursors); + Py_XDECREF(self->blobs); + Py_TYPE(self)->tp_free((PyObject*)self); } @@ -327,6 +332,84 @@ PyObject* pysqlite_connection_cursor(pysqlite_Connection* self, PyObject* args, return cursor; } +PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, + PyObject *kwargs) +{ + static char *kwlist[] = {"table", "column", "row", "readonly", + "dbname", NULL, NULL}; + int rc; + const char *dbname = "main", *table, *column; + long long row; + int readonly = 0; + sqlite3_blob *blob; + pysqlite_Blob *pyblob = NULL; + PyObject *weakref; + + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|ps", kwlist, + &table, &column, &row, &readonly, + &dbname)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_open(self->db, dbname, table, column, row, + !readonly, &blob); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK) { + _pysqlite_seterror(self->db, NULL); + return NULL; + } + + pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + if (!pyblob) { + goto error; + } + + rc = pysqlite_blob_init(pyblob, self, blob); + if (rc) { + Py_CLEAR(pyblob); + goto error; + } + + // Add our blob to connection blobs list + weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); + if (!weakref) { + Py_CLEAR(pyblob); + goto error; + } + if (PyList_Append(self->blobs, weakref) != 0) { + Py_CLEAR(weakref); + Py_CLEAR(pyblob); + goto error; + } + Py_DECREF(weakref); + + return (PyObject*)pyblob; + +error: + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + return NULL; +} + +static void pysqlite_close_all_blobs(pysqlite_Connection *self) +{ + int i; + PyObject *weakref; + PyObject *blob; + + for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + weakref = PyList_GET_ITEM(self->blobs, i); + blob = PyWeakref_GetObject(weakref); + if (blob != Py_None) { + pysqlite_blob_close((pysqlite_Blob*)blob); + } + } +} + PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) { int rc; @@ -337,6 +420,8 @@ PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) pysqlite_do_all_statements(self, ACTION_FINALIZE, 1); + pysqlite_close_all_blobs(self); + if (self->db) { rc = SQLITE3_CLOSE(self->db); @@ -1768,6 +1853,8 @@ static PyGetSetDef connection_getset[] = { static PyMethodDef connection_methods[] = { {"cursor", (PyCFunction)(void(*)(void))pysqlite_connection_cursor, METH_VARARGS|METH_KEYWORDS, PyDoc_STR("Return a cursor for the connection.")}, + {"open_blob", (PyCFunction)pysqlite_connection_blob, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("return a blob object")}, {"close", (PyCFunction)pysqlite_connection_close, METH_NOARGS, PyDoc_STR("Closes the connection.")}, {"commit", (PyCFunction)pysqlite_connection_commit, METH_NOARGS, diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 206085e00a00c7..708a199087c002 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -66,9 +66,10 @@ typedef struct pysqlite_Cache* statement_cache; - /* Lists of weak references to statements and cursors used within this connection */ + /* Lists of weak references to statements, blobs and cursors used within this connection */ PyObject* statements; PyObject* cursors; + PyObject* blobs; /* Counters for how many statements/cursors were created in the connection. May be * reset to 0 at certain intervals */ diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 71d951ee887e47..f138e02357753f 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -28,6 +28,7 @@ #include "prepare_protocol.h" #include "microprotocols.h" #include "row.h" +#include "blob.h" #if SQLITE_VERSION_NUMBER >= 3003003 #define HAVE_SHARED_CACHE @@ -368,7 +369,8 @@ PyMODINIT_FUNC PyInit__sqlite3(void) (pysqlite_connection_setup_types() < 0) || (pysqlite_cache_setup_types() < 0) || (pysqlite_statement_setup_types() < 0) || - (pysqlite_prepare_protocol_setup_types() < 0) + (pysqlite_prepare_protocol_setup_types() < 0) || + (pysqlite_blob_setup_types() < 0) ) { Py_XDECREF(module); return NULL; diff --git a/setup.py b/setup.py index 21a5a58981fc15..8c70025489e181 100644 --- a/setup.py +++ b/setup.py @@ -1523,7 +1523,8 @@ def detect_sqlite(self): '_sqlite/prepare_protocol.c', '_sqlite/row.c', '_sqlite/statement.c', - '_sqlite/util.c', ] + '_sqlite/util.c', + '_sqlite/blob.c' ] sqlite_defines = [] if not MS_WINDOWS: From 3fe91083c9b015ec326279a37e8c297e49747d61 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:11:32 +0200 Subject: [PATCH 02/77] Note that blob size cannot be changed using the blob object --- Doc/library/sqlite3.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ceb92b3fe90fab..e464192e136c13 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -309,6 +309,11 @@ Connection Objects When *readonly* is :const:`True` the BLOB is opened with read permissions. Otherwise the BLOB has read and write permissions. + .. note:: + + The BLOB size cannot be changed using the :class:`Blob` class. Use + `zeroblob` to create the blob in the wanted size in advance. + .. versionadded:: 3.7 .. method:: commit() From 865c1c84be443770f489138dd718f9087a080ac4 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:22:03 +0200 Subject: [PATCH 03/77] Use assertRaises in tests --- Lib/sqlite3/test/dbapi.py | 119 ++++++-------------------------------- 1 file changed, 17 insertions(+), 102 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 85db83ee51f246..8b2fc70bce3abf 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -546,22 +546,12 @@ def CheckSeekFromBlobEnd(self): self.assertEqual(self.blob.tell(), 90) def CheckBlobSeekOverBlobSize(self): - try: + with self.assertRaises(ValueError): self.blob.seek(1000) - self.fail("should have raised a ValueError") - except ValueError: - pass - except Exception: - self.fail("should have raised a ValueError") def CheckBlobSeekUnderBlobSize(self): - try: + with self.assertRaises(ValueError): self.blob.seek(-10) - self.fail("should have raised a ValueError") - except ValueError: - pass - except Exception: - self.fail("should have raised a ValueError") def CheckBlobRead(self): self.assertEqual(self.blob.read(), self.blob_data) @@ -594,81 +584,41 @@ def CheckBlobWriteAdvanceOffset(self): self.assertEqual(self.blob.tell(), 50) def CheckBlobWriteMoreThenBlobSize(self): - try: + with self.assertRaises(sqlite.OperationalError): self.blob.write(b"a" * 1000) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobReadAfterRowChange(self): self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") - try: + with self.assertRaises(sqlite.OperationalError): self.blob.read() - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobWriteAfterRowChange(self): self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") - try: + with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobWriteWhenReadOnly(self): read_only_blob = \ self.cx.open_blob("test", "blob_col", 1, readonly=True) - try: + with self.assertRaises(sqlite.OperationalError): read_only_blob.write(b"aaa") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") read_only_blob.close() def CheckBlobOpenWithBadDb(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadTable(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("notexisintg", "blob_col", 1) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadColumn(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "notexisting", 1) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadRow(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 2) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") @unittest.skipUnless(threading, 'This test requires threading.') @@ -935,13 +885,8 @@ def CheckClosedBlobRead(self): con.execute("insert into test(blob_col) values (zeroblob(100))") blob = con.open_blob("test", "blob_col", 1) con.close() - try: + with self.assertRaises(sqlite.ProgrammingError): blob.read() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except: - self.fail("Should have raised a ProgrammingError") def CheckClosedCreateFunction(self): con = sqlite.connect(":memory:") @@ -1109,57 +1054,32 @@ def tearDown(self): def CheckClosedRead(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.read() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedWrite(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.write(b"aaaaaaaaa") - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedSeek(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.seek(10) - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedTell(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.tell() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedClose(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.close() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") class BlobContextManagerTests(unittest.TestCase): @@ -1180,13 +1100,8 @@ def CheckContextExecute(self): def CheckContextCloseBlob(self): with self.cx.open_blob("test", "blob_col", 1) as blob: blob.seek(10) - try: + with self.assertRaises(sqlite.ProgrammingError): blob.close() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def suite(): From 788fd54d56e52df55fdfff33b481b032beeb449b Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:38:43 +0200 Subject: [PATCH 04/77] Fix doc error --- Doc/library/sqlite3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e464192e136c13..6afef818e4ad70 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -312,7 +312,7 @@ Connection Objects .. note:: The BLOB size cannot be changed using the :class:`Blob` class. Use - `zeroblob` to create the blob in the wanted size in advance. + ``zeroblob`` to create the blob in the wanted size in advance. .. versionadded:: 3.7 From a1361e5ed71cbd1ef461a5eb09dd695a9e66619d Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:26:39 +0200 Subject: [PATCH 05/77] blob support sequence protocol --- Lib/sqlite3/test/dbapi.py | 56 ++++++ Modules/_sqlite/blob.c | 396 ++++++++++++++++++++++++++++++++++---- 2 files changed, 416 insertions(+), 36 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 8b2fc70bce3abf..24122a0cf58062 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -620,6 +620,62 @@ def CheckBlobOpenWithBadRow(self): with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 2) + def CheckBlobGetItem(self): + self.assertEqual(self.blob[5], b"a") + + def CheckBlobGetItemIndexOutOfRange(self): + with self.assertRaises(IndexError): + self.blob[105] + with self.assertRaises(IndexError): + self.blob[-105] + + def CheckBlobGetItemNegativeIndex(self): + self.assertEqual(self.blob[-5], b"a") + + def CheckBlobGetItemInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[b"a"] + + def CheckBlobGetSlice(self): + self.assertEqual(self.blob[5:10], b"aaaaa") + + def CheckBlobGetSliceNegativeIndex(self): + self.assertEqual(self.blob[5:-5], self.blob_data[5:-5]) + + def CheckBlobGetSliceInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[5:b"a"] + + def CheckBlobGetSliceWithSkip(self): + self.blob.write(b"abcdefghij") + self.assertEqual(self.blob[0:10:2], b"acegi") + + def CheckBlobSetItem(self): + self.blob[0] = b"b" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"b" + self.blob_data[1:]) + + def CheckBlobSetSlice(self): + self.blob[0:5] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bbbbb" + self.blob_data[5:]) + + def CheckBlobSetSliceWithSkip(self): + self.blob[0:10:2] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bababababa" + self.blob_data[10:]) + + def CheckBlobGetEmptySlice(self): + self.assertEqual(self.blob[5:5], b"") + + def CheckBlobSetSliceWrongLength(self): + with self.assertRaises(IndexError): + self.blob[5:10] = b"a" + + def CheckBlobConcatNotSupported(self): + with self.assertRaises(SystemError): + self.blob + self.blob + + def CheckBlobRepeateNotSupported(self): + with self.assertRaises(SystemError): + self.blob * 5 @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 11da2b0ad5d691..333c65425a1f0f 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -105,14 +105,43 @@ static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) return blob_length; }; +static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) +{ + PyObject *buffer; + char *raw_buffer; + int rc; + + buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (!buffer) { + return NULL; + } + raw_buffer = PyBytes_AS_STRING(buffer); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + Py_DECREF(buffer); + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return NULL; + } + return buffer; +} + PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) { int read_length = -1; int blob_length = 0; PyObject *buffer; - char *raw_buffer; - int rc; if (!PyArg_ParseTuple(args, "|i", &read_length)) { return NULL; @@ -138,34 +167,36 @@ PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) read_length = blob_length - self->offset; } - buffer = PyBytes_FromStringAndSize(NULL, read_length); - if (!buffer) { - return NULL; + buffer = inner_read(self, read_length, self->offset); + + if (buffer != NULL) { + /* update offset on sucess. */ + self->offset += read_length; } - raw_buffer = PyBytes_AS_STRING(buffer); + + return buffer; +}; + +static int write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +{ + int rc; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + rc = sqlite3_blob_write(self->blob, buf, len, offset); Py_END_ALLOW_THREADS - - if (rc != SQLITE_OK){ - Py_DECREF(buffer); + if (rc != SQLITE_OK) { /* For some reason after modifying blob the - error is not set on the connection db. */ + error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(pysqlite_OperationalError, "Cannot operate on modified blob"); } else { _pysqlite_seterror(self->connection->db, NULL); } - return NULL; + return -1; } - - /* update offset. */ - self->offset += read_length; - - return buffer; -}; + return 0; +} PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) @@ -191,26 +222,15 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) /* TODO: throw better error on data bigger then blob. */ - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_write(self->blob, data_buffer.buf, - data_buffer.len, self->offset); - Py_END_ALLOW_THREADS - if (rc != SQLITE_OK) { - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(pysqlite_OperationalError, - "Cannot operate on modified blob"); - } else { - _pysqlite_seterror(self->connection->db, NULL); - } + rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); + + if (rc == 0) { + self->offset += (int)data_buffer.len; + Py_RETURN_NONE; + } else { PyBuffer_Release(&data_buffer); return NULL; } - - self->offset += (int)data_buffer.len; - PyBuffer_Release(&data_buffer); - Py_RETURN_NONE; } @@ -300,6 +320,300 @@ PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) Py_RETURN_FALSE; } +static PyObject* pysqlite_blob_concat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support concatenation"); + } + return NULL; +} + +static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support repeat operation"); + } + return NULL; +} + +static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) +{ + int blob_length = 0; + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return NULL; + } + + return inner_read(self, 1, i); +} + +static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) +{ + int blob_length = 0; + const char *buf; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(v) && PyBytes_Size(v)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(v); + return write_inner(self, buf, 1, i); +} + + +static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) +{ + int blob_length = 0; + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if (i < 0) + i += blob_length; + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return NULL; + } + // TODO: I am not sure... + return inner_read(self, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + + if (PySlice_GetIndicesEx(item, blob_length, + &start, &stop, &step, &slicelen) < 0) { + return NULL; + } + + if (slicelen <= 0) + return PyBytes_FromStringAndSize("", 0); + else if (step == 1) + return inner_read(self, slicelen, start); + else { + char *result_buf = (char *)PyMem_Malloc(slicelen); + char *data_buff = NULL; + Py_ssize_t cur, i; + PyObject *result; + int rc; + + if (result_buf == NULL) + return PyErr_NoMemory(); + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyMem_Free(result_buf); + return PyErr_NoMemory(); + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return NULL; + } + + for (cur = 0, i = 0; i < slicelen; + cur += step, i++) { + result_buf[i] = data_buff[cur]; + } + result = PyBytes_FromStringAndSize(result_buf, + slicelen); + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return result; + } + } + else { + PyErr_SetString(PyExc_TypeError, + "Blob indices must be integers"); + return NULL; + } +} + + +static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +{ + int blob_length = 0; + int rc; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + const char *buf; + + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += blob_length; + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(value); + return write_inner(self, buf, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + Py_buffer vbuf; + + if (PySlice_GetIndicesEx(item, + blob_length, &start, &stop, + &step, &slicelen) < 0) { + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support slice deletion"); + return -1; + } + if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) + return -1; + if (vbuf.len != slicelen) { + PyErr_SetString(PyExc_IndexError, + "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; + } + + if (slicelen == 0) { + } + else if (step == 1) { + rc = write_inner(self, vbuf.buf, slicelen, start); + } + else { + Py_ssize_t cur, i; + char *data_buff; + + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyErr_NoMemory(); + return -1; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + + for (cur = 0, i = 0; + i < slicelen; + cur += step, i++) + { + data_buff[cur] = ((char *)vbuf.buf)[i]; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + rc = 0; + + } + PyBuffer_Release(&vbuf); + return rc; + } + else { + PyErr_SetString(PyExc_TypeError, + "mmap indices must be integer"); + return -1; + } +} + static PyMethodDef blob_methods[] = { {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, @@ -320,9 +634,18 @@ static PyMethodDef blob_methods[] = { }; static PySequenceMethods blob_sequence_methods = { - (lenfunc)pysqlite_blob_length, /* sq_length */ + .sq_length = (lenfunc)pysqlite_blob_length, + .sq_concat = (binaryfunc)pysqlite_blob_concat, + .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, + .sq_item = (ssizeargfunc)pysqlite_blob_item, + .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, }; +static PyMappingMethods blob_mapping_methods = { + (lenfunc)pysqlite_blob_length, + (binaryfunc)pysqlite_blob_subscript, + (objobjargproc)pysqlite_blob_ass_subscript, +}; PyTypeObject pysqlite_BlobType = { PyVarObject_HEAD_INIT(NULL, 0) @@ -330,6 +653,7 @@ PyTypeObject pysqlite_BlobType = { .tp_basicsize = sizeof(pysqlite_Blob), .tp_dealloc = (destructor)pysqlite_blob_dealloc, .tp_as_sequence = &blob_sequence_methods, + .tp_as_mapping = &blob_mapping_methods, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), .tp_methods = blob_methods, From 5aedfba9a1541473e72faaaaabf3d63e0b972ad6 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:34:59 +0200 Subject: [PATCH 06/77] Calculate blob length once at creation --- Modules/_sqlite/blob.c | 73 ++++++++++++------------------------------ Modules/_sqlite/blob.h | 1 + 2 files changed, 21 insertions(+), 53 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 333c65425a1f0f..abe8cac1b271bc 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,6 +11,10 @@ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, self->blob = blob; self->in_weakreflist = NULL; + Py_BEGIN_ALLOW_THREADS + self->length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + if (!pysqlite_check_thread(self->connection)) { return -1; } @@ -94,15 +98,11 @@ PyObject* pysqlite_blob_close(pysqlite_Blob *self) static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) { - int blob_length; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - return blob_length; + return self->length; }; static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) @@ -140,7 +140,6 @@ static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) { int read_length = -1; - int blob_length = 0; PyObject *buffer; if (!PyArg_ParseTuple(args, "|i", &read_length)) { @@ -151,20 +150,14 @@ PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) return NULL; } - - /* TODO: make this multithreaded and safe! */ - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (read_length < 0) { /* same as file read. */ - read_length = blob_length; + read_length = self->length; } /* making sure we don't read more then blob size */ - if (read_length > blob_length - self->offset) { - read_length = blob_length - self->offset; + if (read_length > self->length - self->offset) { + read_length = self->length - self->offset; } buffer = inner_read(self, read_length, self->offset); @@ -236,7 +229,7 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) { - int blob_length, offset, from_what = 0; + int offset, from_what = 0; if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { return NULL; @@ -247,10 +240,6 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - switch (from_what) { case 0: // relative to blob begin break; @@ -261,17 +250,17 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->offset + offset; break; case 2: // relative to blob end - if (offset > INT_MAX - blob_length) { + if (offset > INT_MAX - self->length) { goto overflow; } - offset = blob_length + offset; + offset = self->length + offset; break; default: return PyErr_Format(PyExc_ValueError, "from_what should be 0, 1 or 2"); } - if (offset < 0 || offset > blob_length) { + if (offset < 0 || offset > self->length) { return PyErr_Format(PyExc_ValueError, "offset out of blob range"); } @@ -340,17 +329,11 @@ static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) { - int blob_length = 0; - if (!pysqlite_check_blob(self)) { return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (i < 0 || i >= blob_length) { + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } @@ -360,18 +343,13 @@ static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) { - int blob_length = 0; const char *buf; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (i < 0 || i >= blob_length) { + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } @@ -393,23 +371,17 @@ static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) { - int blob_length = 0; - if (!pysqlite_check_blob(self)) { return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) - i += blob_length; - if (i < 0 || i >= blob_length) { + i += self->length; + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; @@ -420,7 +392,7 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, blob_length, + if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, &slicelen) < 0) { return NULL; } @@ -484,17 +456,12 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) { - int blob_length = 0; int rc; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); const char *buf; @@ -502,8 +469,8 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) - i += blob_length; - if (i < 0 || i >= blob_length) { + i += self->length; + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; @@ -527,7 +494,7 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb Py_buffer vbuf; if (PySlice_GetIndicesEx(item, - blob_length, &start, &stop, + self->length, &start, &stop, &step, &slicelen) < 0) { return -1; } diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 204918fd4f01c6..649f09e5ecca24 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -10,6 +10,7 @@ typedef struct pysqlite_Connection* connection; sqlite3_blob *blob; int offset; + int length; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Blob; From db6ef327457f9eb56e448c50309b714322212eed Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:41:20 +0200 Subject: [PATCH 07/77] Add initial Doc for sequence protocol --- Doc/library/sqlite3.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 6afef818e4ad70..068e15bf2534b9 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -878,7 +878,9 @@ Blob Objects .. class:: Blob A :class:`Blob` instance can read and write the data in the - :abbr:`BLOB (Binary Large OBject)`. + :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both + the file and sequence protocol. For example You can read data from the + :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. .. method:: Blob.close() From 219f4cb4a1b3db1783f4e9c272cd3e6de4028908 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:04:22 +0300 Subject: [PATCH 08/77] Don't support blob operation --- Lib/sqlite3/test/dbapi.py | 4 ++++ Modules/_sqlite/blob.c | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 24122a0cf58062..d769d62446b2b7 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -677,6 +677,10 @@ def CheckBlobRepeateNotSupported(self): with self.assertRaises(SystemError): self.blob * 5 + def CheckBlobContainsNotSupported(self): + with self.assertRaises(SystemError): + b"aaaaa" in self.blob + @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index abe8cac1b271bc..7b2432491450a4 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -327,6 +327,15 @@ static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) return NULL; } +static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support cotains operation"); + } + return -1; +} + static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) { if (!pysqlite_check_blob(self)) { @@ -606,6 +615,7 @@ static PySequenceMethods blob_sequence_methods = { .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, .sq_item = (ssizeargfunc)pysqlite_blob_item, .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, + .sq_contains = (objobjproc)pysqlite_blob_contains, }; static PyMappingMethods blob_mapping_methods = { From 6dafe0e7b5d651f373304f39961c5a5552e05531 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:16:26 +0300 Subject: [PATCH 09/77] move news entry to blurb --- .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst new file mode 100644 index 00000000000000..4f7b2fefed8039 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -0,0 +1,4 @@ +The :class:`sqlite3.Connection` now has the +:meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` +allows incremental I/O operations to blobs. (Contributed by Aviv Palivoda in +:issue:`24905`) From ffac9013a89c68561b51443585a799aacad3b6e7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:32:15 +0300 Subject: [PATCH 10/77] Add blob to PCBuild --- PCbuild/_sqlite3.vcxproj | 2 ++ PCbuild/_sqlite3.vcxproj.filters | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 7e0062692b8f83..55f46c963b1e7d 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -107,6 +107,7 @@ + @@ -118,6 +119,7 @@ + diff --git a/PCbuild/_sqlite3.vcxproj.filters b/PCbuild/_sqlite3.vcxproj.filters index 51830f6a4451a4..8d26c9ab6eb43f 100644 --- a/PCbuild/_sqlite3.vcxproj.filters +++ b/PCbuild/_sqlite3.vcxproj.filters @@ -39,6 +39,9 @@ Header Files + + Header Files + @@ -68,6 +71,9 @@ Source Files + + Source Files + From 3475fa1c422a7418dc1a7765ad33479307afa531 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 15:30:45 +0300 Subject: [PATCH 11/77] Update version --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 068e15bf2534b9..9b6d0472089063 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -314,7 +314,7 @@ Connection Objects The BLOB size cannot be changed using the :class:`Blob` class. Use ``zeroblob`` to create the blob in the wanted size in advance. - .. versionadded:: 3.7 + .. versionadded:: 3.8 .. method:: commit() @@ -873,7 +873,7 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.7 +.. versionadded:: 3.8 .. class:: Blob From f6015ff3b39343898066e70c879d17dc5a78f89d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 15:32:14 +0300 Subject: [PATCH 12/77] Fix memory leak --- Modules/_sqlite/blob.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7b2432491450a4..d560563d3e766d 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -219,6 +219,7 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) if (rc == 0) { self->offset += (int)data_buffer.len; + PyBuffer_Release(&data_buffer); Py_RETURN_NONE; } else { PyBuffer_Release(&data_buffer); From 01e526ca5c94b480380919a79e320037bc7f65f0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 29 Jul 2020 19:40:23 +0300 Subject: [PATCH 13/77] Update version --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9b6d0472089063..841b2be4fdacf8 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -314,7 +314,7 @@ Connection Objects The BLOB size cannot be changed using the :class:`Blob` class. Use ``zeroblob`` to create the blob in the wanted size in advance. - .. versionadded:: 3.8 + .. versionadded:: 3.10 .. method:: commit() @@ -873,7 +873,7 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.8 +.. versionadded:: 3.10 .. class:: Blob From 354bebf2dc561a6f079fc430edf8d9eb7cab6496 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:01:26 +0300 Subject: [PATCH 14/77] Fix CR comments in documentation and testing --- Doc/includes/sqlite3/blob.py | 6 +++--- Doc/includes/sqlite3/blob_with.py | 6 +++--- Doc/library/sqlite3.rst | 5 +++-- Lib/sqlite3/test/dbapi.py | 8 +++++--- .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index b8d2c78d1a365d..18514876af6191 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -6,8 +6,8 @@ con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle blob = con.open_blob("test", "blob_col", 1, 1) -blob.write(b"a" * 5) -blob.write(b"b" * 5) +blob.write(b"Hello") +blob.write(b"World") blob.seek(0) -print(blob.read()) # will print b"aaaaabbbbb" +print(blob.read()) # will print b"HelloWorld" blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index 624b680591901a..f220518c09a1ab 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -6,7 +6,7 @@ con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle with con.open_blob("test", "blob_col", 1, 1) as blob: - blob.write(b"a" * 5) - blob.write(b"b" * 5) + blob.write(b"Hello") + blob.write(b"World") blob.seek(0) - print(blob.read()) # will print b"aaaaabbbbb" + print(blob.read()) # will print b"HelloWorld" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 841b2be4fdacf8..b754481ad48757 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -879,8 +879,9 @@ Blob Objects A :class:`Blob` instance can read and write the data in the :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both - the file and sequence protocol. For example You can read data from the + the file and sequence protocol. For example, you can read data from the :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. + You can call ``len(obj)`` to get size of the BLOB. .. method:: Blob.close() @@ -911,7 +912,7 @@ Blob Objects Return the current offset of the BLOB. - .. method:: Blob.seek(offset, [whence]) + .. method:: Blob.seek(offset, whence=os.SEEK_SET) Set the BLOB offset. The *whence* argument is optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index d769d62446b2b7..a611feb56414eb 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -1177,9 +1177,11 @@ def suite(): blob_suite = unittest.makeSuite(BlobTests, "Check") closed_blob_suite = unittest.makeSuite(ClosedBlobTests, "Check") blob_context_manager_suite = unittest.makeSuite(BlobContextManagerTests, "Check") - return unittest.TestSuite((module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, - ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, - blob_suite, closed_blob_suite, blob_context_manager_suite)) + return unittest.TestSuite(( + module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, + ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, + blob_suite, closed_blob_suite, blob_context_manager_suite, + )) def test(): runner = unittest.TextTestRunner() diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst index 4f7b2fefed8039..c7d2405d89539e 100644 --- a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -1,4 +1,4 @@ The :class:`sqlite3.Connection` now has the :meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` -allows incremental I/O operations to blobs. (Contributed by Aviv Palivoda in +allows incremental I/O operations to blobs. (Patch by Aviv Palivoda in :issue:`24905`) From 9709456b864e48e1cc3a3e838b311806a9ca6acb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:08:58 +0300 Subject: [PATCH 15/77] Fix CR comments in code --- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/connection.h | 2 +- setup.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index b99fcdb24fc25f..7e112b7fdb705a 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -336,7 +336,7 @@ PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = {"table", "column", "row", "readonly", - "dbname", NULL, NULL}; + "dbname", NULL}; int rc; const char *dbname = "main", *table, *column; long long row; diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 708a199087c002..52dc27c8adff2d 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -66,7 +66,7 @@ typedef struct pysqlite_Cache* statement_cache; - /* Lists of weak references to statements, blobs and cursors used within this connection */ + /* Lists of weak references to statements, cursors and blobs used within this connection */ PyObject* statements; PyObject* cursors; PyObject* blobs; diff --git a/setup.py b/setup.py index 8c70025489e181..dd6b3509247fd8 100644 --- a/setup.py +++ b/setup.py @@ -1524,7 +1524,8 @@ def detect_sqlite(self): '_sqlite/row.c', '_sqlite/statement.c', '_sqlite/util.c', - '_sqlite/blob.c' ] + '_sqlite/blob.c', + ] sqlite_defines = [] if not MS_WINDOWS: From e6e5099d5a5d9a4e69317208a4a463922a903fd5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:23:21 +0300 Subject: [PATCH 16/77] Make readonly and dbname keyword arguements only --- Doc/includes/sqlite3/blob.py | 2 +- Doc/includes/sqlite3/blob_with.py | 2 +- Doc/library/sqlite3.rst | 2 +- Modules/_sqlite/connection.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index 18514876af6191..afd7812a8b3af9 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -5,7 +5,7 @@ con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle -blob = con.open_blob("test", "blob_col", 1, 1) +blob = con.open_blob("test", "blob_col", 1) blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index f220518c09a1ab..fdca9fbc638ea2 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -5,7 +5,7 @@ con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle -with con.open_blob("test", "blob_col", 1, 1) as blob: +with con.open_blob("test", "blob_col", 1) as blob: blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index b754481ad48757..827c2e0588611a 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -301,7 +301,7 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. - .. method:: open_blob(table, column, row, readonly=False, dbname="main") + .. method:: open_blob(table, column, row, *, readonly=False, dbname="main") On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 7e112b7fdb705a..1d64bae7ca98f6 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -346,7 +346,7 @@ PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, PyObject *weakref; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|ps", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|$ps", kwlist, &table, &column, &row, &readonly, &dbname)) { return NULL; From e9a80800f0883de7e3b9fb34ca1b8172fd745460 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:32:47 +0300 Subject: [PATCH 17/77] Fix more CR comments --- Modules/_sqlite/blob.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index d560563d3e766d..f222d5cd477f22 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -257,19 +257,19 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->length + offset; break; default: - return PyErr_Format(PyExc_ValueError, + return PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); } if (offset < 0 || offset > self->length) { - return PyErr_Format(PyExc_ValueError, "offset out of blob range"); + return PyErr_SetString(PyExc_ValueError, "offset out of blob range"); } self->offset = offset; Py_RETURN_NONE; overflow: - return PyErr_Format(PyExc_OverflowError, "seek offset result in overflow"); + return PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); } @@ -332,7 +332,7 @@ static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) { if (pysqlite_check_blob(self)) { PyErr_SetString(PyExc_SystemError, - "Blob don't support cotains operation"); + "Blob don't support contains operation"); } return -1; } @@ -407,11 +407,11 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) return NULL; } - if (slicelen <= 0) + if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); - else if (step == 1) + } else if (step == 1) { return inner_read(self, slicelen, start); - else { + } else { char *result_buf = (char *)PyMem_Malloc(slicelen); char *data_buff = NULL; Py_ssize_t cur, i; @@ -586,7 +586,7 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb } else { PyErr_SetString(PyExc_TypeError, - "mmap indices must be integer"); + "Blob indices must be integer"); return -1; } } From d4fb1b5afd50091d99ecaa4022df109626d7325e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:57:07 +0300 Subject: [PATCH 18/77] Add more indicative error on write bigger then blob length --- Lib/sqlite3/test/dbapi.py | 2 +- Modules/_sqlite/blob.c | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index a611feb56414eb..fd88faa7765e02 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -584,7 +584,7 @@ def CheckBlobWriteAdvanceOffset(self): self.assertEqual(self.blob.tell(), 50) def CheckBlobWriteMoreThenBlobSize(self): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaises(ValueError): self.blob.write(b"a" * 1000) def CheckBlobReadAfterRowChange(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f222d5cd477f22..49e664631e2774 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -208,12 +208,17 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) return NULL; } - if (!pysqlite_check_blob(self)) { + if (data_buffer.len > self->length - self->offset) { + PyErr_SetString(PyExc_ValueError, + "data longer than blob length"); PyBuffer_Release(&data_buffer); return NULL; } - /* TODO: throw better error on data bigger then blob. */ + if (!pysqlite_check_blob(self)) { + PyBuffer_Release(&data_buffer); + return NULL; + } rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); @@ -257,19 +262,22 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->length + offset; break; default: - return PyErr_SetString(PyExc_ValueError, + PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); + return NULL; } if (offset < 0 || offset > self->length) { - return PyErr_SetString(PyExc_ValueError, "offset out of blob range"); + PyErr_SetString(PyExc_ValueError, "offset out of blob range"); + return NULL; } self->offset = offset; Py_RETURN_NONE; overflow: - return PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); + PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); + return NULL; } From 525a9c3e6c7418309054e812fd7fd2bbf358a981 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 11:44:29 +0200 Subject: [PATCH 19/77] Adapt sqlite3.Connection.open_blob() to AC --- Modules/_sqlite/clinic/connection.c.h | 97 ++++++++++++++++++++++++++- Modules/_sqlite/connection.c | 35 +++++----- 2 files changed, 115 insertions(+), 17 deletions(-) diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 315d163dde668f..e14f94a8d04038 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -140,6 +140,101 @@ pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ return return_value; } +PyDoc_STRVAR(pysqlite_connection_open_blob__doc__, +"open_blob($self, /, table, column, row, *, readonly=False,\n" +" dbname=\'main\')\n" +"--\n" +"\n" +"Return a blob object. Non-standard."); + +#define PYSQLITE_CONNECTION_OPEN_BLOB_METHODDEF \ + {"open_blob", (PyCFunction)(void(*)(void))pysqlite_connection_open_blob, METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_open_blob__doc__}, + +static PyObject * +pysqlite_connection_open_blob_impl(pysqlite_Connection *self, + const char *table, const char *column, + int row, int readonly, const char *dbname); + +static PyObject * +pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"table", "column", "row", "readonly", "dbname", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "open_blob", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + const char *table; + const char *column; + int row; + int readonly = 0; + const char *dbname = "main"; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("open_blob", "argument 'table'", "str", args[0]); + goto exit; + } + Py_ssize_t table_length; + table = PyUnicode_AsUTF8AndSize(args[0], &table_length); + if (table == NULL) { + goto exit; + } + if (strlen(table) != (size_t)table_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("open_blob", "argument 'column'", "str", args[1]); + goto exit; + } + Py_ssize_t column_length; + column = PyUnicode_AsUTF8AndSize(args[1], &column_length); + if (column == NULL) { + goto exit; + } + if (strlen(column) != (size_t)column_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + row = _PyLong_AsInt(args[2]); + if (row == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + readonly = _PyLong_AsInt(args[3]); + if (readonly == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (!PyUnicode_Check(args[4])) { + _PyArg_BadArgument("open_blob", "argument 'dbname'", "str", args[4]); + goto exit; + } + Py_ssize_t dbname_length; + dbname = PyUnicode_AsUTF8AndSize(args[4], &dbname_length); + if (dbname == NULL) { + goto exit; + } + if (strlen(dbname) != (size_t)dbname_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } +skip_optional_kwonly: + return_value = pysqlite_connection_open_blob_impl(self, table, column, row, readonly, dbname); + +exit: + return return_value; +} + PyDoc_STRVAR(pysqlite_connection_close__doc__, "close($self, /)\n" "--\n" @@ -816,4 +911,4 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */ -/*[clinic end generated code: output=9c0dfc6c1ebf9039 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c37726d47594c3d input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index b8a951fa2f6ed5..37e320356bc901 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -374,26 +374,30 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) return cursor; } -PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, - PyObject *kwargs) +/*[clinic input] +_sqlite3.Connection.open_blob as pysqlite_connection_open_blob + + table: str + column: str + row: int + * + readonly: bool(accept={int}) = False + dbname: str = "main" + +Return a blob object. Non-standard. +[clinic start generated code]*/ + +static PyObject * +pysqlite_connection_open_blob_impl(pysqlite_Connection *self, + const char *table, const char *column, + int row, int readonly, const char *dbname) +/*[clinic end generated code: output=54dfadc6f1283245 input=38c93f0f7d73a1ef]*/ { - static char *kwlist[] = {"table", "column", "row", "readonly", - "dbname", NULL}; int rc; - const char *dbname = "main", *table, *column; - long long row; - int readonly = 0; sqlite3_blob *blob; pysqlite_Blob *pyblob = NULL; PyObject *weakref; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|$ps", kwlist, - &table, &column, &row, &readonly, - &dbname)) { - return NULL; - } - Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_open(self->db, dbname, table, column, row, !readonly, &blob); @@ -1957,8 +1961,6 @@ static PyGetSetDef connection_getset[] = { }; static PyMethodDef connection_methods[] = { - {"open_blob", (PyCFunction)pysqlite_connection_blob, METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("return a blob object")}, PYSQLITE_CONNECTION_BACKUP_METHODDEF PYSQLITE_CONNECTION_CLOSE_METHODDEF PYSQLITE_CONNECTION_COMMIT_METHODDEF @@ -1975,6 +1977,7 @@ static PyMethodDef connection_methods[] = { PYSQLITE_CONNECTION_INTERRUPT_METHODDEF PYSQLITE_CONNECTION_ITERDUMP_METHODDEF PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF + PYSQLITE_CONNECTION_OPEN_BLOB_METHODDEF PYSQLITE_CONNECTION_ROLLBACK_METHODDEF PYSQLITE_CONNECTION_SET_AUTHORIZER_METHODDEF PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF From 2f65cc886fd2e5b69e218590d7094767d2b72170 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:05:41 +0200 Subject: [PATCH 20/77] Adapt sqlite.Blob to AC --- Modules/_sqlite/blob.c | 159 +++++++++++++++++-------- Modules/_sqlite/blob.h | 1 + Modules/_sqlite/clinic/blob.c.h | 202 ++++++++++++++++++++++++++++++++ Modules/_sqlite/connection.c | 15 --- 4 files changed, 316 insertions(+), 61 deletions(-) create mode 100644 Modules/_sqlite/clinic/blob.c.h diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7fe768f62b4816..3ab801cc05e26a 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -1,6 +1,15 @@ #include "blob.h" #include "util.h" +#define clinic_state() (pysqlite_get_state(NULL)) +#include "clinic/blob.c.h" +#undef clinic_state + +/*[clinic input] +module _sqlite3 +class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, sqlite3_blob *blob) @@ -85,7 +94,15 @@ int pysqlite_check_blob(pysqlite_Blob *blob) } -PyObject* pysqlite_blob_close(pysqlite_Blob *self) +/*[clinic input] +_sqlite3.Blob.close as blob_close + +Close blob. +[clinic start generated code]*/ + +static PyObject * +blob_close_impl(pysqlite_Blob *self) +/*[clinic end generated code: output=848accc20a138d1b input=56c86df5cab22490]*/ { if (!pysqlite_check_blob(self)) { @@ -96,6 +113,21 @@ PyObject* pysqlite_blob_close(pysqlite_Blob *self) Py_RETURN_NONE; }; +void +pysqlite_close_all_blobs(pysqlite_Connection *self) +{ + int i; + PyObject *weakref; + PyObject *blob; + + for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + weakref = PyList_GET_ITEM(self->blobs, i); + blob = PyWeakref_GetObject(weakref); + if (blob != Py_None) { + blob_close_impl((pysqlite_Blob*)blob); + } + } +} static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) { @@ -138,15 +170,21 @@ static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) } -PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) +/*[clinic input] +_sqlite3.Blob.read as blob_read + + read_length: int = -1 + / + +Read data from blob. +[clinic start generated code]*/ + +static PyObject * +blob_read_impl(pysqlite_Blob *self, int read_length) +/*[clinic end generated code: output=9c4881a77860b216 input=753a766082129348]*/ { - int read_length = -1; PyObject *buffer; - if (!PyArg_ParseTuple(args, "|i", &read_length)) { - return NULL; - } - if (!pysqlite_check_blob(self)) { return NULL; } @@ -193,56 +231,62 @@ static int write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int } -PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) +/*[clinic input] +_sqlite3.Blob.write as blob_write + + data_buffer: Py_buffer + / + +Write data to blob. +[clinic start generated code]*/ + +static PyObject * +blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer) +/*[clinic end generated code: output=dc8da6900b969799 input=8597402caf368add]*/ { - Py_buffer data_buffer; int rc; - if (PyObject_GetBuffer(data, &data_buffer, PyBUF_SIMPLE) < 0) { - return NULL; - } - - if (data_buffer.len > INT_MAX) { + if (data_buffer->len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); - PyBuffer_Release(&data_buffer); return NULL; } - if (data_buffer.len > self->length - self->offset) { + if (data_buffer->len > self->length - self->offset) { PyErr_SetString(PyExc_ValueError, "data longer than blob length"); - PyBuffer_Release(&data_buffer); return NULL; } if (!pysqlite_check_blob(self)) { - PyBuffer_Release(&data_buffer); return NULL; } - rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); + rc = write_inner(self, data_buffer->buf, data_buffer->len, self->offset); if (rc == 0) { - self->offset += (int)data_buffer.len; - PyBuffer_Release(&data_buffer); + self->offset += (int)data_buffer->len; Py_RETURN_NONE; } else { - PyBuffer_Release(&data_buffer); return NULL; } } -PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) -{ - int offset, from_what = 0; +/*[clinic input] +_sqlite3.Blob.seek as blob_seek - if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { - return NULL; - } + offset: int + from_what: int = 0 + / +Change the access position for a blob. +[clinic start generated code]*/ +static PyObject * +blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) +/*[clinic end generated code: output=c7f07cd9e1d90bf7 input=f8dae77055129be3]*/ +{ if (!pysqlite_check_blob(self)) { return NULL; } @@ -282,7 +326,15 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) } -PyObject* pysqlite_blob_tell(pysqlite_Blob *self) +/*[clinic input] +_sqlite3.Blob.tell as blob_tell + +Return current access position for a blob. +[clinic start generated code]*/ + +static PyObject * +blob_tell_impl(pysqlite_Blob *self) +/*[clinic end generated code: output=3d3ba484a90b3a99 input=aa1660f9aee18be4]*/ { if (!pysqlite_check_blob(self)) { return NULL; @@ -292,7 +344,15 @@ PyObject* pysqlite_blob_tell(pysqlite_Blob *self) } -PyObject* pysqlite_blob_enter(pysqlite_Blob *self) +/*[clinic input] +_sqlite3.Blob.__enter__ as blob_enter + +Blob context manager enter. +[clinic start generated code]*/ + +static PyObject * +blob_enter_impl(pysqlite_Blob *self) +/*[clinic end generated code: output=4fd32484b071a6cd input=fe4842c3c582d5a7]*/ { if (!pysqlite_check_blob(self)) { return NULL; @@ -303,14 +363,28 @@ PyObject* pysqlite_blob_enter(pysqlite_Blob *self) } -PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) +/*[clinic input] +_sqlite3.Blob.__exit__ as blob_exit + + type: object + val: object + tb: object + / + +Blob context manager exit. +[clinic start generated code]*/ + +static PyObject * +blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, + PyObject *tb) +/*[clinic end generated code: output=fc86ceeb2b68c7b2 input=575d9ecea205f35f]*/ { PyObject *res; if (!pysqlite_check_blob(self)) { return NULL; } - res = pysqlite_blob_close(self); + res = blob_close_impl(self); if (!res) { return NULL; } @@ -602,20 +676,13 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb static PyMethodDef blob_methods[] = { - {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, - PyDoc_STR("read data from blob")}, - {"write", (PyCFunction)pysqlite_blob_write, METH_O, - PyDoc_STR("write data to blob")}, - {"close", (PyCFunction)pysqlite_blob_close, METH_NOARGS, - PyDoc_STR("close blob")}, - {"seek", (PyCFunction)pysqlite_blob_seek, METH_VARARGS, - PyDoc_STR("change blob current offset")}, - {"tell", (PyCFunction)pysqlite_blob_tell, METH_NOARGS, - PyDoc_STR("return blob current offset")}, - {"__enter__", (PyCFunction)pysqlite_blob_enter, METH_NOARGS, - PyDoc_STR("blob context manager enter")}, - {"__exit__", (PyCFunction)pysqlite_blob_exit, METH_VARARGS, - PyDoc_STR("blob context manager exit")}, + BLOB_CLOSE_METHODDEF + BLOB_ENTER_METHODDEF + BLOB_EXIT_METHODDEF + BLOB_READ_METHODDEF + BLOB_SEEK_METHODDEF + BLOB_TELL_METHODDEF + BLOB_WRITE_METHODDEF {NULL, NULL} }; diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 649f09e5ecca24..bae8e4f088b1e0 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -22,5 +22,6 @@ int pysqlite_blob_init(pysqlite_Blob* self, pysqlite_Connection* connection, PyObject* pysqlite_blob_close(pysqlite_Blob *self); int pysqlite_blob_setup_types(void); +void pysqlite_close_all_blobs(pysqlite_Connection *self); #endif diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h new file mode 100644 index 00000000000000..84bad030650a79 --- /dev/null +++ b/Modules/_sqlite/clinic/blob.c.h @@ -0,0 +1,202 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(blob_close__doc__, +"close($self, /)\n" +"--\n" +"\n" +"Close blob."); + +#define BLOB_CLOSE_METHODDEF \ + {"close", (PyCFunction)blob_close, METH_NOARGS, blob_close__doc__}, + +static PyObject * +blob_close_impl(pysqlite_Blob *self); + +static PyObject * +blob_close(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +{ + return blob_close_impl(self); +} + +PyDoc_STRVAR(blob_read__doc__, +"read($self, read_length=-1, /)\n" +"--\n" +"\n" +"Read data from blob."); + +#define BLOB_READ_METHODDEF \ + {"read", (PyCFunction)(void(*)(void))blob_read, METH_FASTCALL, blob_read__doc__}, + +static PyObject * +blob_read_impl(pysqlite_Blob *self, int read_length); + +static PyObject * +blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int read_length = -1; + + if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + read_length = _PyLong_AsInt(args[0]); + if (read_length == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = blob_read_impl(self, read_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(blob_write__doc__, +"write($self, data_buffer, /)\n" +"--\n" +"\n" +"Write data to blob."); + +#define BLOB_WRITE_METHODDEF \ + {"write", (PyCFunction)blob_write, METH_O, blob_write__doc__}, + +static PyObject * +blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer); + +static PyObject * +blob_write(pysqlite_Blob *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer data_buffer = {NULL, NULL}; + + if (PyObject_GetBuffer(arg, &data_buffer, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!PyBuffer_IsContiguous(&data_buffer, 'C')) { + _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); + goto exit; + } + return_value = blob_write_impl(self, &data_buffer); + +exit: + /* Cleanup for data_buffer */ + if (data_buffer.obj) { + PyBuffer_Release(&data_buffer); + } + + return return_value; +} + +PyDoc_STRVAR(blob_seek__doc__, +"seek($self, offset, from_what=0, /)\n" +"--\n" +"\n" +"Change the access position for a blob."); + +#define BLOB_SEEK_METHODDEF \ + {"seek", (PyCFunction)(void(*)(void))blob_seek, METH_FASTCALL, blob_seek__doc__}, + +static PyObject * +blob_seek_impl(pysqlite_Blob *self, int offset, int from_what); + +static PyObject * +blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int offset; + int from_what = 0; + + if (!_PyArg_CheckPositional("seek", nargs, 1, 2)) { + goto exit; + } + offset = _PyLong_AsInt(args[0]); + if (offset == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + from_what = _PyLong_AsInt(args[1]); + if (from_what == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = blob_seek_impl(self, offset, from_what); + +exit: + return return_value; +} + +PyDoc_STRVAR(blob_tell__doc__, +"tell($self, /)\n" +"--\n" +"\n" +"Return current access position for a blob."); + +#define BLOB_TELL_METHODDEF \ + {"tell", (PyCFunction)blob_tell, METH_NOARGS, blob_tell__doc__}, + +static PyObject * +blob_tell_impl(pysqlite_Blob *self); + +static PyObject * +blob_tell(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +{ + return blob_tell_impl(self); +} + +PyDoc_STRVAR(blob_enter__doc__, +"__enter__($self, /)\n" +"--\n" +"\n" +"Blob context manager enter."); + +#define BLOB_ENTER_METHODDEF \ + {"__enter__", (PyCFunction)blob_enter, METH_NOARGS, blob_enter__doc__}, + +static PyObject * +blob_enter_impl(pysqlite_Blob *self); + +static PyObject * +blob_enter(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +{ + return blob_enter_impl(self); +} + +PyDoc_STRVAR(blob_exit__doc__, +"__exit__($self, type, val, tb, /)\n" +"--\n" +"\n" +"Blob context manager exit."); + +#define BLOB_EXIT_METHODDEF \ + {"__exit__", (PyCFunction)(void(*)(void))blob_exit, METH_FASTCALL, blob_exit__doc__}, + +static PyObject * +blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, + PyObject *tb); + +static PyObject * +blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *type; + PyObject *val; + PyObject *tb; + + if (!_PyArg_CheckPositional("__exit__", nargs, 3, 3)) { + goto exit; + } + type = args[0]; + val = args[1]; + tb = args[2]; + return_value = blob_exit_impl(self, type, val, tb); + +exit: + return return_value; +} +/*[clinic end generated code: output=5d378130443aa9ce input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 37e320356bc901..789098614ac702 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -441,21 +441,6 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } -static void pysqlite_close_all_blobs(pysqlite_Connection *self) -{ - int i; - PyObject *weakref; - PyObject *blob; - - for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { - weakref = PyList_GET_ITEM(self->blobs, i); - blob = PyWeakref_GetObject(weakref); - if (blob != Py_None) { - pysqlite_blob_close((pysqlite_Blob*)blob); - } - } -} - /*[clinic input] _sqlite3.Connection.close as pysqlite_connection_close From 0dd047fcfbb3868384a35b51f129781a02e59b5c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:31:32 +0200 Subject: [PATCH 21/77] PEP 7 and normalised naming --- Modules/_sqlite/blob.c | 312 +++++++++++++++++------------------ Modules/_sqlite/blob.h | 11 +- Modules/_sqlite/connection.c | 18 +- 3 files changed, 162 insertions(+), 179 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 3ab801cc05e26a..2c5750b5b18bb7 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,8 +11,9 @@ class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ -int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, - sqlite3_blob *blob) +int +pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, + sqlite3_blob *blob) { Py_INCREF(connection); self->connection = connection; @@ -30,45 +31,43 @@ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, return 0; } -static void remove_blob_from_connection_blob_list(pysqlite_Blob *self) +static void +remove_blob_from_connection(pysqlite_Blob *self) { - Py_ssize_t i; - PyObject *item; - - for (i = 0; i < PyList_GET_SIZE(self->connection->blobs); i++) { - item = PyList_GET_ITEM(self->connection->blobs, i); + Py_ssize_t size = PyList_GET_SIZE(self->connection->blobs); + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *item = PyList_GET_ITEM(self->connection->blobs, i); if (PyWeakref_GetObject(item) == (PyObject *)self) { - PyList_SetSlice(self->connection->blobs, i, i+1, NULL); + Py_ssize_t low = i; + Py_ssize_t high = low + 1; + PyList_SetSlice(self->connection->blobs, low, high, NULL); break; } } } -static void _close_blob_inner(pysqlite_Blob* self) +static void +close_blob(pysqlite_Blob *self) { - sqlite3_blob *blob; - - /* close the blob */ - blob = self->blob; - self->blob = NULL; - if (blob) { + if (self->blob) { Py_BEGIN_ALLOW_THREADS - sqlite3_blob_close(blob); + sqlite3_blob_close(self->blob); Py_END_ALLOW_THREADS + self->blob = NULL; } - /* remove from connection weaklist */ - remove_blob_from_connection_blob_list(self); + remove_blob_from_connection(self); if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)self); + PyObject_ClearWeakRefs((PyObject *)self); } } -static void pysqlite_blob_dealloc(pysqlite_Blob* self) +static void +blob_dealloc(pysqlite_Blob *self) { - _close_blob_inner(self); + close_blob(self); Py_XDECREF(self->connection); - Py_TYPE(self)->tp_free((PyObject*)self); + Py_TYPE(self)->tp_free((PyObject *)self); } @@ -77,20 +76,20 @@ static void pysqlite_blob_dealloc(pysqlite_Blob* self) * * 0 => error; 1 => ok */ -int pysqlite_check_blob(pysqlite_Blob *blob) +static int +check_blob(pysqlite_Blob *blob) { - - if (!blob->blob) { + if (blob->blob == NULL) { pysqlite_state *state = pysqlite_get_state(NULL); PyErr_SetString(state->ProgrammingError, "Cannot operate on a closed blob."); return 0; - } else if (!pysqlite_check_connection(blob->connection) || - !pysqlite_check_thread(blob->connection)) { + } + else if (!pysqlite_check_connection(blob->connection) || + !pysqlite_check_thread(blob->connection)) { return 0; - } else { - return 1; } + return 1; } @@ -104,64 +103,57 @@ static PyObject * blob_close_impl(pysqlite_Blob *self) /*[clinic end generated code: output=848accc20a138d1b input=56c86df5cab22490]*/ { - - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } - - _close_blob_inner(self); + close_blob(self); Py_RETURN_NONE; }; void pysqlite_close_all_blobs(pysqlite_Connection *self) { - int i; - PyObject *weakref; - PyObject *blob; - - for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { - weakref = PyList_GET_ITEM(self->blobs, i); - blob = PyWeakref_GetObject(weakref); + for (int i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + PyObject *weakref = PyList_GET_ITEM(self->blobs, i); + PyObject *blob = PyWeakref_GetObject(weakref); if (blob != Py_None) { - blob_close_impl((pysqlite_Blob*)blob); + blob_close_impl((pysqlite_Blob *)blob); } } } -static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) +static Py_ssize_t +blob_length(pysqlite_Blob *self) { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return -1; } - return self->length; }; -static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) +static PyObject * +inner_read(pysqlite_Blob *self, int read_length, int offset) { - PyObject *buffer; - char *raw_buffer; - int rc; - - buffer = PyBytes_FromStringAndSize(NULL, read_length); - if (!buffer) { + PyObject *buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (buffer == NULL) { return NULL; } - raw_buffer = PyBytes_AS_STRING(buffer); + char *raw_buffer = PyBytes_AS_STRING(buffer); + int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); Py_END_ALLOW_THREADS - if (rc != SQLITE_OK){ + if (rc != SQLITE_OK) { Py_DECREF(buffer); /* For some reason after modifying blob the error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { + } + else { _pysqlite_seterror(self->connection->state, self->connection->db); } return NULL; @@ -183,9 +175,7 @@ static PyObject * blob_read_impl(pysqlite_Blob *self, int read_length) /*[clinic end generated code: output=9c4881a77860b216 input=753a766082129348]*/ { - PyObject *buffer; - - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -199,8 +189,7 @@ blob_read_impl(pysqlite_Blob *self, int read_length) read_length = self->length - self->offset; } - buffer = inner_read(self, read_length, self->offset); - + PyObject *buffer = inner_read(self, read_length, self->offset); if (buffer != NULL) { /* update offset on sucess. */ self->offset += read_length; @@ -209,20 +198,23 @@ blob_read_impl(pysqlite_Blob *self, int read_length) return buffer; }; -static int write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +static int +write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) { int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_write(self->blob, buf, len, offset); Py_END_ALLOW_THREADS + if (rc != SQLITE_OK) { /* For some reason after modifying blob the error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { + } + else { _pysqlite_seterror(self->connection->state, self->connection->db); } return -1; @@ -247,18 +239,16 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer) int rc; if (data_buffer->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "data longer than INT_MAX bytes"); + PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); return NULL; } if (data_buffer->len > self->length - self->offset) { - PyErr_SetString(PyExc_ValueError, - "data longer than blob length"); + PyErr_SetString(PyExc_ValueError, "data longer than blob length"); return NULL; } - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -287,7 +277,7 @@ static PyObject * blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) /*[clinic end generated code: output=c7f07cd9e1d90bf7 input=f8dae77055129be3]*/ { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -307,8 +297,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) offset = self->length + offset; break; default: - PyErr_SetString(PyExc_ValueError, - "from_what should be 0, 1 or 2"); + PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); return NULL; } @@ -336,10 +325,9 @@ static PyObject * blob_tell_impl(pysqlite_Blob *self) /*[clinic end generated code: output=3d3ba484a90b3a99 input=aa1660f9aee18be4]*/ { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } - return PyLong_FromLong(self->offset); } @@ -354,7 +342,7 @@ static PyObject * blob_enter_impl(pysqlite_Blob *self) /*[clinic end generated code: output=4fd32484b071a6cd input=fe4842c3c582d5a7]*/ { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -380,7 +368,7 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, /*[clinic end generated code: output=fc86ceeb2b68c7b2 input=575d9ecea205f35f]*/ { PyObject *res; - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -393,36 +381,39 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, Py_RETURN_FALSE; } -static PyObject* pysqlite_blob_concat(pysqlite_Blob *self, PyObject *args) +static PyObject * +blob_concat(pysqlite_Blob *self, PyObject *args) { - if (pysqlite_check_blob(self)) { - PyErr_SetString(PyExc_SystemError, - "Blob don't support concatenation"); + if (check_blob(self)) { + PyErr_SetString(PyExc_SystemError, "Blob don't support concatenation"); } return NULL; } -static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) +static PyObject * +blob_repeat(pysqlite_Blob *self, PyObject *args) { - if (pysqlite_check_blob(self)) { + if (check_blob(self)) { PyErr_SetString(PyExc_SystemError, "Blob don't support repeat operation"); } return NULL; } -static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) +static int +blob_contains(pysqlite_Blob *self, PyObject *args) { - if (pysqlite_check_blob(self)) { + if (check_blob(self)) { PyErr_SetString(PyExc_SystemError, "Blob don't support contains operation"); } return -1; } -static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) +static PyObject * +blob_item(pysqlite_Blob *self, Py_ssize_t i) { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -434,11 +425,10 @@ static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) return inner_read(self, 1, i); } -static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) +static int +blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) { - const char *buf; - - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return -1; } @@ -457,26 +447,28 @@ static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v return -1; } - buf = PyBytes_AsString(v); + const char *buf = PyBytes_AsString(v); return write_inner(self, buf, 1, i); } -static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) +static PyObject * +blob_subscript(pysqlite_Blob *self, PyObject *item) { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) + if (i == -1 && PyErr_Occurred()) { return NULL; - if (i < 0) + } + if (i < 0) { i += self->length; + } if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, - "Blob index out of range"); + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } // TODO: I am not sure... @@ -485,87 +477,85 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, + &slicelen) < 0) { return NULL; } if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); - } else if (step == 1) { + } + else if (step == 1) { return inner_read(self, slicelen, start); - } else { + } + else { char *result_buf = (char *)PyMem_Malloc(slicelen); - char *data_buff = NULL; - Py_ssize_t cur, i; - PyObject *result; - int rc; - - if (result_buf == NULL) + if (result_buf == NULL) { return PyErr_NoMemory(); + } - data_buff = (char *)PyMem_Malloc(stop - start); + char *data_buff = (char *)PyMem_Malloc(stop - start); if (data_buff == NULL) { PyMem_Free(result_buf); return PyErr_NoMemory(); } + int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); Py_END_ALLOW_THREADS - if (rc != SQLITE_OK){ + if (rc != SQLITE_OK) { /* For some reason after modifying blob the error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { - _pysqlite_seterror(self->connection->state, self->connection->db); + } + else { + _pysqlite_seterror(self->connection->state, + self->connection->db); } PyMem_Free(result_buf); PyMem_Free(data_buff); return NULL; } - for (cur = 0, i = 0; i < slicelen; - cur += step, i++) { + for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { result_buf[i] = data_buff[cur]; } - result = PyBytes_FromStringAndSize(result_buf, - slicelen); + PyObject *result = PyBytes_FromStringAndSize(result_buf, slicelen); PyMem_Free(result_buf); PyMem_Free(data_buff); return result; } } else { - PyErr_SetString(PyExc_TypeError, - "Blob indices must be integers"); + PyErr_SetString(PyExc_TypeError, "Blob indices must be integers"); return NULL; } } -static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +static int +blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) { int rc; - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return -1; } if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - const char *buf; - - if (i == -1 && PyErr_Occurred()) + if (i == -1 && PyErr_Occurred()) { return -1; - if (i < 0) + } + if (i < 0) { i += self->length; + } if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, - "Blob index out of range"); + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } if (value == NULL) { @@ -579,43 +569,39 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb return -1; } - buf = PyBytes_AsString(value); + const char *buf = PyBytes_AsString(value); return write_inner(self, buf, 1, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - Py_buffer vbuf; - - if (PySlice_GetIndicesEx(item, - self->length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, + &slicelen) < 0) { return -1; } if (value == NULL) { PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support slice deletion"); + "Blob object doesn't support slice deletion"); return -1; } - if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) + + Py_buffer vbuf; + if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { return -1; + } if (vbuf.len != slicelen) { PyErr_SetString(PyExc_IndexError, - "Blob slice assignment is wrong size"); + "Blob slice assignment is wrong size"); PyBuffer_Release(&vbuf); return -1; } - if (slicelen == 0) { + if (slicelen == 0) { // FIXME } else if (step == 1) { rc = write_inner(self, vbuf.buf, slicelen, start); } else { - Py_ssize_t cur, i; - char *data_buff; - - - data_buff = (char *)PyMem_Malloc(stop - start); + char *data_buff = (char *)PyMem_Malloc(stop - start); if (data_buff == NULL) { PyErr_NoMemory(); return -1; @@ -631,17 +617,15 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { + } + else { _pysqlite_seterror(self->connection->state, self->connection->db); } PyMem_Free(data_buff); rc = -1; } - for (cur = 0, i = 0; - i < slicelen; - cur += step, i++) - { + for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { data_buff[cur] = ((char *)vbuf.buf)[i]; } @@ -655,7 +639,8 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { + } + else { _pysqlite_seterror(self->connection->state, self->connection->db); } PyMem_Free(data_buff); @@ -687,33 +672,34 @@ static PyMethodDef blob_methods[] = { }; static PySequenceMethods blob_sequence_methods = { - .sq_length = (lenfunc)pysqlite_blob_length, - .sq_concat = (binaryfunc)pysqlite_blob_concat, - .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, - .sq_item = (ssizeargfunc)pysqlite_blob_item, - .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, - .sq_contains = (objobjproc)pysqlite_blob_contains, + .sq_length = (lenfunc)blob_length, + .sq_concat = (binaryfunc)blob_concat, + .sq_repeat = (ssizeargfunc)blob_repeat, + .sq_item = (ssizeargfunc)blob_item, + .sq_ass_item = (ssizeobjargproc)blob_ass_item, + .sq_contains = (objobjproc)blob_contains, }; static PyMappingMethods blob_mapping_methods = { - (lenfunc)pysqlite_blob_length, - (binaryfunc)pysqlite_blob_subscript, - (objobjargproc)pysqlite_blob_ass_subscript, + (lenfunc)blob_length, + (binaryfunc)blob_subscript, + (objobjargproc)blob_ass_subscript, }; PyTypeObject pysqlite_BlobType = { - PyVarObject_HEAD_INIT(NULL, 0) - MODULE_NAME ".Blob", - .tp_basicsize = sizeof(pysqlite_Blob), - .tp_dealloc = (destructor)pysqlite_blob_dealloc, - .tp_as_sequence = &blob_sequence_methods, - .tp_as_mapping = &blob_mapping_methods, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), - .tp_methods = blob_methods, + PyVarObject_HEAD_INIT(NULL, 0) + MODULE_NAME ".Blob", + .tp_basicsize = sizeof(pysqlite_Blob), + .tp_dealloc = (destructor)blob_dealloc, + .tp_as_sequence = &blob_sequence_methods, + .tp_as_mapping = &blob_mapping_methods, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), + .tp_methods = blob_methods, }; -extern int pysqlite_blob_setup_types(void) +extern int +pysqlite_blob_setup_types(void) { pysqlite_BlobType.tp_new = PyType_GenericNew; return PyType_Ready(&pysqlite_BlobType); diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index bae8e4f088b1e0..b55dcc9dc7b5bb 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -4,22 +4,21 @@ #include "sqlite3.h" #include "connection.h" -typedef struct -{ +typedef struct { PyObject_HEAD - pysqlite_Connection* connection; + pysqlite_Connection *connection; sqlite3_blob *blob; int offset; int length; - PyObject* in_weakreflist; /* List of weak references */ + PyObject *in_weakreflist; } pysqlite_Blob; extern PyTypeObject pysqlite_BlobType; -int pysqlite_blob_init(pysqlite_Blob* self, pysqlite_Connection* connection, +int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, sqlite3_blob *blob); -PyObject* pysqlite_blob_close(pysqlite_Blob *self); +PyObject *pysqlite_blob_close(pysqlite_Blob *self); int pysqlite_blob_setup_types(void); void pysqlite_close_all_blobs(pysqlite_Connection *self); diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 789098614ac702..4ebecb9fc639b7 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -395,12 +395,10 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, { int rc; sqlite3_blob *blob; - pysqlite_Blob *pyblob = NULL; - PyObject *weakref; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_open(self->db, dbname, table, column, row, - !readonly, &blob); + rc = sqlite3_blob_open(self->db, dbname, table, column, row, !readonly, + &blob); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -408,24 +406,24 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } - pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); - if (!pyblob) { + pysqlite_Blob *pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + if (pyblob == NULL) { goto error; } rc = pysqlite_blob_init(pyblob, self, blob); - if (rc) { + if (rc < 0) { Py_CLEAR(pyblob); goto error; } // Add our blob to connection blobs list - weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); - if (!weakref) { + PyObject *weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); + if (weakref == NULL) { Py_CLEAR(pyblob); goto error; } - if (PyList_Append(self->blobs, weakref) != 0) { + if (PyList_Append(self->blobs, weakref) < 0) { Py_CLEAR(weakref); Py_CLEAR(pyblob); goto error; From d34a77bb4bde0c71cd3564ccbd101fb99d72a9f1 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:33:16 +0200 Subject: [PATCH 22/77] Use Py_NewRef and Py_IsNone --- Modules/_sqlite/blob.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 2c5750b5b18bb7..9d9bde5e303672 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -15,8 +15,7 @@ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, sqlite3_blob *blob) { - Py_INCREF(connection); - self->connection = connection; + self->connection = (pysqlite_Connection *)Py_NewRef(connection); self->offset = 0; self->blob = blob; self->in_weakreflist = NULL; @@ -116,7 +115,7 @@ pysqlite_close_all_blobs(pysqlite_Connection *self) for (int i = 0; i < PyList_GET_SIZE(self->blobs); i++) { PyObject *weakref = PyList_GET_ITEM(self->blobs, i); PyObject *blob = PyWeakref_GetObject(weakref); - if (blob != Py_None) { + if (!Py_IsNone(blob)) { blob_close_impl((pysqlite_Blob *)blob); } } @@ -345,9 +344,7 @@ blob_enter_impl(pysqlite_Blob *self) if (!check_blob(self)) { return NULL; } - - Py_INCREF(self); - return (PyObject *)self; + return Py_NewRef(self); } From 9d557054307581bbbf202077eeee07f2470cb713 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:35:25 +0200 Subject: [PATCH 23/77] Harden blob_open() --- Modules/_sqlite/connection.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 4ebecb9fc639b7..eadff8ac8b8845 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -413,26 +413,24 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, rc = pysqlite_blob_init(pyblob, self, blob); if (rc < 0) { - Py_CLEAR(pyblob); goto error; } // Add our blob to connection blobs list - PyObject *weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); + PyObject *weakref = PyWeakref_NewRef((PyObject *)pyblob, NULL); if (weakref == NULL) { - Py_CLEAR(pyblob); goto error; } - if (PyList_Append(self->blobs, weakref) < 0) { - Py_CLEAR(weakref); - Py_CLEAR(pyblob); + rc = PyList_Append(self->blobs, weakref); + Py_DECREF(weakref); + if (rc < 0) { goto error; } - Py_DECREF(weakref); - return (PyObject*)pyblob; + return (PyObject *)pyblob; error: + Py_XDECREF(pyblob); Py_BEGIN_ALLOW_THREADS sqlite3_blob_close(blob); Py_END_ALLOW_THREADS From d47ea17c573527c6e40aa772c45f57487208f604 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:52:40 +0200 Subject: [PATCH 24/77] Move blob init to blob_open() --- Lib/sqlite3/test/dbapi.py | 2 +- Modules/_sqlite/blob.c | 18 ----------- Modules/_sqlite/blob.h | 2 -- Modules/_sqlite/clinic/connection.c.h | 39 ++++++++++++------------ Modules/_sqlite/connection.c | 43 ++++++++++++++++----------- 5 files changed, 46 insertions(+), 58 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 51ddc8564dc109..f2f51a7891d760 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -824,7 +824,7 @@ def test_blob_write_when_readonly(self): def test_blob_open_with_bad_db(self): with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") + self.cx.open_blob("test", "blob_col", 1, name="notexisintg") def test_blob_open_with_bad_table(self): with self.assertRaises(sqlite.OperationalError): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 9d9bde5e303672..c11f27d047d529 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,24 +11,6 @@ class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ -int -pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, - sqlite3_blob *blob) -{ - self->connection = (pysqlite_Connection *)Py_NewRef(connection); - self->offset = 0; - self->blob = blob; - self->in_weakreflist = NULL; - - Py_BEGIN_ALLOW_THREADS - self->length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (!pysqlite_check_thread(self->connection)) { - return -1; - } - return 0; -} static void remove_blob_from_connection(pysqlite_Blob *self) diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index b55dcc9dc7b5bb..462cd1386e8a92 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -16,8 +16,6 @@ typedef struct { extern PyTypeObject pysqlite_BlobType; -int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, - sqlite3_blob *blob); PyObject *pysqlite_blob_close(pysqlite_Blob *self); int pysqlite_blob_setup_types(void); diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index e14f94a8d04038..5e0c2157c72df9 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -141,8 +141,7 @@ pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ } PyDoc_STRVAR(pysqlite_connection_open_blob__doc__, -"open_blob($self, /, table, column, row, *, readonly=False,\n" -" dbname=\'main\')\n" +"open_blob($self, table, column, row, /, *, readonly=False, name=\'main\')\n" "--\n" "\n" "Return a blob object. Non-standard."); @@ -152,29 +151,29 @@ PyDoc_STRVAR(pysqlite_connection_open_blob__doc__, static PyObject * pysqlite_connection_open_blob_impl(pysqlite_Connection *self, - const char *table, const char *column, - int row, int readonly, const char *dbname); + const char *table, const char *col, + int row, int readonly, const char *name); static PyObject * pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"table", "column", "row", "readonly", "dbname", NULL}; + static const char * const _keywords[] = {"", "", "", "readonly", "name", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "open_blob", 0}; PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; const char *table; - const char *column; + const char *col; int row; int readonly = 0; - const char *dbname = "main"; + const char *name = "main"; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); if (!args) { goto exit; } if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("open_blob", "argument 'table'", "str", args[0]); + _PyArg_BadArgument("open_blob", "argument 1", "str", args[0]); goto exit; } Py_ssize_t table_length; @@ -187,15 +186,15 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, goto exit; } if (!PyUnicode_Check(args[1])) { - _PyArg_BadArgument("open_blob", "argument 'column'", "str", args[1]); + _PyArg_BadArgument("open_blob", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t column_length; - column = PyUnicode_AsUTF8AndSize(args[1], &column_length); - if (column == NULL) { + Py_ssize_t col_length; + col = PyUnicode_AsUTF8AndSize(args[1], &col_length); + if (col == NULL) { goto exit; } - if (strlen(column) != (size_t)column_length) { + if (strlen(col) != (size_t)col_length) { PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } @@ -216,20 +215,20 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, } } if (!PyUnicode_Check(args[4])) { - _PyArg_BadArgument("open_blob", "argument 'dbname'", "str", args[4]); + _PyArg_BadArgument("open_blob", "argument 'name'", "str", args[4]); goto exit; } - Py_ssize_t dbname_length; - dbname = PyUnicode_AsUTF8AndSize(args[4], &dbname_length); - if (dbname == NULL) { + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[4], &name_length); + if (name == NULL) { goto exit; } - if (strlen(dbname) != (size_t)dbname_length) { + if (strlen(name) != (size_t)name_length) { PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } skip_optional_kwonly: - return_value = pysqlite_connection_open_blob_impl(self, table, column, row, readonly, dbname); + return_value = pysqlite_connection_open_blob_impl(self, table, col, row, readonly, name); exit: return return_value; @@ -911,4 +910,4 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */ -/*[clinic end generated code: output=2c37726d47594c3d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a392a91a234a8fc8 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index eadff8ac8b8845..ee7298090a91cb 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -378,27 +378,34 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) _sqlite3.Connection.open_blob as pysqlite_connection_open_blob table: str - column: str + column as col: str row: int + / * readonly: bool(accept={int}) = False - dbname: str = "main" + name: str = "main" Return a blob object. Non-standard. [clinic start generated code]*/ static PyObject * pysqlite_connection_open_blob_impl(pysqlite_Connection *self, - const char *table, const char *column, - int row, int readonly, const char *dbname) -/*[clinic end generated code: output=54dfadc6f1283245 input=38c93f0f7d73a1ef]*/ + const char *table, const char *col, + int row, int readonly, const char *name) +/*[clinic end generated code: output=5c11d18e7eb629ef input=9192dd28146d4e14]*/ { - int rc; + if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { + return NULL; + } + + int rc, len; sqlite3_blob *blob; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_open(self->db, dbname, table, column, row, !readonly, - &blob); + rc = sqlite3_blob_open(self->db, name, table, col, row, !readonly, &blob); + if (rc == SQLITE_OK) { + len = sqlite3_blob_bytes(blob); + } Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -406,18 +413,19 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } - pysqlite_Blob *pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); - if (pyblob == NULL) { + pysqlite_Blob *obj = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + if (obj == NULL) { goto error; } - rc = pysqlite_blob_init(pyblob, self, blob); - if (rc < 0) { - goto error; - } + obj->connection = (pysqlite_Connection *)Py_NewRef(self); + obj->blob = blob; + obj->offset = 0; + obj->length = len; + obj->in_weakreflist = NULL; // Add our blob to connection blobs list - PyObject *weakref = PyWeakref_NewRef((PyObject *)pyblob, NULL); + PyObject *weakref = PyWeakref_NewRef((PyObject *)obj, NULL); if (weakref == NULL) { goto error; } @@ -427,10 +435,11 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, goto error; } - return (PyObject *)pyblob; + return (PyObject *)obj; error: - Py_XDECREF(pyblob); + Py_XDECREF(obj); + Py_BEGIN_ALLOW_THREADS sqlite3_blob_close(blob); Py_END_ALLOW_THREADS From e07a116a5ed86c97763e0a91d641870646cd4be1 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:54:25 +0200 Subject: [PATCH 25/77] initialise rc in blob_ass_subscript() --- Modules/_sqlite/blob.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index c11f27d047d529..7fd00ad1affca3 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -519,7 +519,7 @@ blob_subscript(pysqlite_Blob *self, PyObject *item) static int blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) { - int rc; + int rc = -1; if (!check_blob(self)) { return -1; @@ -574,7 +574,7 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } - if (slicelen == 0) { // FIXME + if (slicelen == 0) { // FIXME } else if (step == 1) { rc = write_inner(self, vbuf.buf, slicelen, start); From 982c81269193abdf09b7eef86eb77546ec3630b7 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 13:04:54 +0200 Subject: [PATCH 26/77] Improve blob.seek() parameter naming --- Modules/_sqlite/blob.c | 10 +++++----- Modules/_sqlite/clinic/blob.c.h | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7fd00ad1affca3..79231385e16396 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -248,21 +248,21 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer) _sqlite3.Blob.seek as blob_seek offset: int - from_what: int = 0 + origin: int = 0 / Change the access position for a blob. [clinic start generated code]*/ static PyObject * -blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) -/*[clinic end generated code: output=c7f07cd9e1d90bf7 input=f8dae77055129be3]*/ +blob_seek_impl(pysqlite_Blob *self, int offset, int origin) +/*[clinic end generated code: output=854c5a0e208547a5 input=cc33da6f28af0561]*/ { if (!check_blob(self)) { return NULL; } - switch (from_what) { + switch (origin) { case 0: // relative to blob begin break; case 1: // relative to current position @@ -278,7 +278,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) offset = self->length + offset; break; default: - PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); + PyErr_SetString(PyExc_ValueError, "'origin' should be 0, 1 or 2"); return NULL; } diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index 84bad030650a79..fa995a4938389b 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -92,7 +92,7 @@ blob_write(pysqlite_Blob *self, PyObject *arg) } PyDoc_STRVAR(blob_seek__doc__, -"seek($self, offset, from_what=0, /)\n" +"seek($self, offset, origin=0, /)\n" "--\n" "\n" "Change the access position for a blob."); @@ -101,14 +101,14 @@ PyDoc_STRVAR(blob_seek__doc__, {"seek", (PyCFunction)(void(*)(void))blob_seek, METH_FASTCALL, blob_seek__doc__}, static PyObject * -blob_seek_impl(pysqlite_Blob *self, int offset, int from_what); +blob_seek_impl(pysqlite_Blob *self, int offset, int origin); static PyObject * blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int offset; - int from_what = 0; + int origin = 0; if (!_PyArg_CheckPositional("seek", nargs, 1, 2)) { goto exit; @@ -120,12 +120,12 @@ blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - from_what = _PyLong_AsInt(args[1]); - if (from_what == -1 && PyErr_Occurred()) { + origin = _PyLong_AsInt(args[1]); + if (origin == -1 && PyErr_Occurred()) { goto exit; } skip_optional: - return_value = blob_seek_impl(self, offset, from_what); + return_value = blob_seek_impl(self, offset, origin); exit: return return_value; @@ -199,4 +199,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5d378130443aa9ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=76c066429020440e input=a9049054013a1b77]*/ From e92495b2b22c5d6d225e1b311c17c395fd15cf72 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 13:40:04 +0200 Subject: [PATCH 27/77] Adapt to heap types and allow calling close multiple times --- Lib/sqlite3/test/dbapi.py | 12 ---- Modules/_sqlite/blob.c | 123 ++++++++++++++++++++--------------- Modules/_sqlite/blob.h | 4 +- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/module.c | 2 +- Modules/_sqlite/module.h | 1 + 6 files changed, 74 insertions(+), 70 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index f2f51a7891d760..3c994c64ae8c33 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -933,12 +933,6 @@ def test_closed_blob_tell(self): with self.assertRaises(sqlite.ProgrammingError): self.blob.tell() - def test_closed_blob_close(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.close() - def test_closed_blob_read(self): con = sqlite.connect(":memory:") con.execute("create table test(id integer primary key, blob_col blob)") @@ -964,12 +958,6 @@ def test_blob_ctx_mgr_execute(self): blob.write(data) self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) - def test_blob_ctx_mgr_close(self): - with self.cx.open_blob("test", "blob_col", 1) as blob: - blob.seek(10) - with self.assertRaises(sqlite.ProgrammingError): - blob.close() - class ThreadTests(unittest.TestCase): def setUp(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 79231385e16396..7a80d7125919af 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -12,21 +12,6 @@ class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ -static void -remove_blob_from_connection(pysqlite_Blob *self) -{ - Py_ssize_t size = PyList_GET_SIZE(self->connection->blobs); - for (Py_ssize_t i = 0; i < size; i++) { - PyObject *item = PyList_GET_ITEM(self->connection->blobs, i); - if (PyWeakref_GetObject(item) == (PyObject *)self) { - Py_ssize_t low = i; - Py_ssize_t high = low + 1; - PyList_SetSlice(self->connection->blobs, low, high, NULL); - break; - } - } -} - static void close_blob(pysqlite_Blob *self) { @@ -36,19 +21,37 @@ close_blob(pysqlite_Blob *self) Py_END_ALLOW_THREADS self->blob = NULL; } +} - remove_blob_from_connection(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)self); - } +static int +blob_traverse(pysqlite_Blob *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->connection); + return 0; +} + +static int +blob_clear(pysqlite_Blob *self) +{ + Py_CLEAR(self->connection); + return 0; } static void blob_dealloc(pysqlite_Blob *self) { + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + close_blob(self); - Py_XDECREF(self->connection); - Py_TYPE(self)->tp_free((PyObject *)self); + + if (self->in_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject*)self); + } + tp->tp_clear((PyObject *)self); + tp->tp_free(self); + Py_DECREF(tp); } @@ -58,18 +61,18 @@ blob_dealloc(pysqlite_Blob *self) * 0 => error; 1 => ok */ static int -check_blob(pysqlite_Blob *blob) +check_blob(pysqlite_Blob *self) { - if (blob->blob == NULL) { + if (!pysqlite_check_connection(self->connection) || + !pysqlite_check_thread(self->connection)) { + return 0; + } + if (self->blob == NULL) { pysqlite_state *state = pysqlite_get_state(NULL); PyErr_SetString(state->ProgrammingError, "Cannot operate on a closed blob."); return 0; } - else if (!pysqlite_check_connection(blob->connection) || - !pysqlite_check_thread(blob->connection)) { - return 0; - } return 1; } @@ -84,7 +87,9 @@ static PyObject * blob_close_impl(pysqlite_Blob *self) /*[clinic end generated code: output=848accc20a138d1b input=56c86df5cab22490]*/ { - if (!check_blob(self)) { + if (!pysqlite_check_connection(self->connection) || + !pysqlite_check_thread(self->connection)) + { return NULL; } close_blob(self); @@ -650,36 +655,48 @@ static PyMethodDef blob_methods[] = { {NULL, NULL} }; -static PySequenceMethods blob_sequence_methods = { - .sq_length = (lenfunc)blob_length, - .sq_concat = (binaryfunc)blob_concat, - .sq_repeat = (ssizeargfunc)blob_repeat, - .sq_item = (ssizeargfunc)blob_item, - .sq_ass_item = (ssizeobjargproc)blob_ass_item, - .sq_contains = (objobjproc)blob_contains, +static struct PyMemberDef blob_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(pysqlite_Blob, in_weakreflist), READONLY}, }; -static PyMappingMethods blob_mapping_methods = { - (lenfunc)blob_length, - (binaryfunc)blob_subscript, - (objobjargproc)blob_ass_subscript, +static PyType_Slot blob_slots[] = { + {Py_tp_dealloc, blob_dealloc}, + {Py_tp_traverse, blob_traverse}, + {Py_tp_clear, blob_clear}, + {Py_tp_methods, blob_methods}, + {Py_tp_members, blob_members}, + + // Sequence protocol + {Py_sq_length, blob_length}, + {Py_sq_concat, blob_concat}, + {Py_sq_repeat, blob_repeat}, + {Py_sq_item, blob_item}, + {Py_sq_ass_item, blob_ass_item}, + {Py_sq_contains, blob_contains}, + + // Mapping protocol + {Py_mp_length, blob_length}, + {Py_mp_subscript, blob_subscript}, + {Py_mp_ass_subscript, blob_ass_subscript}, + {0, NULL}, }; -PyTypeObject pysqlite_BlobType = { - PyVarObject_HEAD_INIT(NULL, 0) - MODULE_NAME ".Blob", - .tp_basicsize = sizeof(pysqlite_Blob), - .tp_dealloc = (destructor)blob_dealloc, - .tp_as_sequence = &blob_sequence_methods, - .tp_as_mapping = &blob_mapping_methods, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), - .tp_methods = blob_methods, +static PyType_Spec blob_spec = { + .name = MODULE_NAME ".Blob", + .basicsize = sizeof(pysqlite_Blob), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = blob_slots, }; -extern int -pysqlite_blob_setup_types(void) +int +pysqlite_blob_setup_types(PyObject *module) { - pysqlite_BlobType.tp_new = PyType_GenericNew; - return PyType_Ready(&pysqlite_BlobType); + PyObject *type = PyType_FromModuleAndSpec(module, &blob_spec, NULL); + if (type == NULL) { + return -1; + } + pysqlite_state *state = pysqlite_get_state(module); + state->BlobType = (PyTypeObject *)type; + return 0; } diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 462cd1386e8a92..cb8a5ebba191b7 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -14,11 +14,9 @@ typedef struct { PyObject *in_weakreflist; } pysqlite_Blob; -extern PyTypeObject pysqlite_BlobType; - PyObject *pysqlite_blob_close(pysqlite_Blob *self); -int pysqlite_blob_setup_types(void); +int pysqlite_blob_setup_types(PyObject *module); void pysqlite_close_all_blobs(pysqlite_Connection *self); #endif diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index ee7298090a91cb..5587037fc6939f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -413,7 +413,7 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } - pysqlite_Blob *obj = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + pysqlite_Blob *obj = PyObject_GC_New(pysqlite_Blob, self->state->BlobType); if (obj == NULL) { goto error; } diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 5a58b47f24637b..2207cb4e086bf7 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -448,7 +448,7 @@ PyMODINIT_FUNC PyInit__sqlite3(void) (pysqlite_connection_setup_types(module) < 0) || (pysqlite_statement_setup_types(module) < 0) || (pysqlite_prepare_protocol_setup_types(module) < 0) || - (pysqlite_blob_setup_types() < 0) + (pysqlite_blob_setup_types(module) < 0) ) { goto error; } diff --git a/Modules/_sqlite/module.h b/Modules/_sqlite/module.h index c273c1f9ed9f29..4c684b2b6b9ea8 100644 --- a/Modules/_sqlite/module.h +++ b/Modules/_sqlite/module.h @@ -53,6 +53,7 @@ typedef struct { int BaseTypeAdapted; int enable_callback_tracebacks; + PyTypeObject *BlobType; PyTypeObject *ConnectionType; PyTypeObject *CursorType; PyTypeObject *PrepareProtocolType; From 8f2ce8a33cf351c33dbc5613c10ef99bdd9a24c4 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 13:41:15 +0200 Subject: [PATCH 28/77] Consolidate tests --- Lib/sqlite3/test/dbapi.py | 83 ++++++++++++--------------------------- 1 file changed, 26 insertions(+), 57 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 3c994c64ae8c33..5615aee476ab0c 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -763,13 +763,10 @@ def test_blob_seek_from_end(self): self.blob.seek(-10, 2) self.assertEqual(self.blob.tell(), 90) - def test_blob_seek_over_size(self): - with self.assertRaises(ValueError): - self.blob.seek(1000) - - def test_blob_seek_under_size(self): - with self.assertRaises(ValueError): - self.blob.seek(-10) + def test_blob_seek_error(self): + for pos in 1000, -10: + with self.subTest(pos=pos): + self.assertRaises(ValueError, self.blob.seek, pos) def test_blob_read(self): self.assertEqual(self.blob.read(), self.blob_data) @@ -899,39 +896,29 @@ def test_blob_contains_not_supported(self): with self.assertRaises(SystemError): b"aaaaa" in self.blob + def test_blob_context_manager(self): + data = b"a" * 100 + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.write(data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) -class ClosedBlobTests(unittest.TestCase): - def setUp(self): - self.cx = sqlite.connect(":memory:") - self.cx.execute("create table test(id integer primary key, blob_col blob)") - self.cx.execute("insert into test(blob_col) values (zeroblob(100))") - - def tearDown(self): - self.cx.close() - - def test_closed_blob_read(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.read() - - def test_closed_blob_write(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.write(b"aaaaaaaaa") - - def test_closed_blob_seek(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.seek(10) - - def test_closed_blob_tell(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.tell() + def test_blob_closed(self): + cx = sqlite.connect(":memory:") + cx.execute("create table test(b blob)") + cx.execute("insert into test values (zeroblob(100))") + blob = cx.open_blob("test", "b", 1) + blob.close() + ops = [ + lambda: blob.read(), + lambda: blob.write(b""), + lambda: blob.seek(0), + lambda: blob.tell(), + ] + msg = "Cannot operate on a closed blob" + for op in ops: + with self.subTest(op=op): + with self.assertRaisesRegex(sqlite.ProgrammingError, msg): + op() def test_closed_blob_read(self): con = sqlite.connect(":memory:") @@ -943,22 +930,6 @@ def test_closed_blob_read(self): blob.read() -class BlobContextManagerTests(unittest.TestCase): - def setUp(self): - self.cx = sqlite.connect(":memory:") - self.cx.execute("create table test(id integer primary key, blob_col blob)") - self.cx.execute("insert into test(blob_col) values (zeroblob(100))") - - def tearDown(self): - self.cx.close() - - def test_blob_ctx_mgr_execute(self): - data = b"a" * 100 - with self.cx.open_blob("test", "blob_col", 1) as blob: - blob.write(data) - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) - - class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -1388,9 +1359,7 @@ def wait(): def suite(): tests = [ - BlobContextManagerTests, BlobTests, - ClosedBlobTests, ClosedConTests, ClosedCurTests, ConnectionTests, From 0a87520c85592cfea15bb8b75fa6d2a27794f8f7 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 13:59:38 +0200 Subject: [PATCH 29/77] Naming: read_length => length --- Modules/_sqlite/blob.c | 24 ++++++++++++------------ Modules/_sqlite/clinic/blob.c.h | 14 +++++++------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7a80d7125919af..a812f6f8fd3712 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -118,9 +118,9 @@ blob_length(pysqlite_Blob *self) }; static PyObject * -inner_read(pysqlite_Blob *self, int read_length, int offset) +inner_read(pysqlite_Blob *self, int length, int offset) { - PyObject *buffer = PyBytes_FromStringAndSize(NULL, read_length); + PyObject *buffer = PyBytes_FromStringAndSize(NULL, length); if (buffer == NULL) { return NULL; } @@ -128,7 +128,7 @@ inner_read(pysqlite_Blob *self, int read_length, int offset) int rc; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + rc = sqlite3_blob_read(self->blob, raw_buffer, length, self->offset); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -151,34 +151,34 @@ inner_read(pysqlite_Blob *self, int read_length, int offset) /*[clinic input] _sqlite3.Blob.read as blob_read - read_length: int = -1 + length: int = -1 / Read data from blob. [clinic start generated code]*/ static PyObject * -blob_read_impl(pysqlite_Blob *self, int read_length) -/*[clinic end generated code: output=9c4881a77860b216 input=753a766082129348]*/ +blob_read_impl(pysqlite_Blob *self, int length) +/*[clinic end generated code: output=1fc99b2541360dde input=b4b443e99af5548f]*/ { if (!check_blob(self)) { return NULL; } - if (read_length < 0) { + if (length < 0) { /* same as file read. */ - read_length = self->length; + length = self->length; } /* making sure we don't read more then blob size */ - if (read_length > self->length - self->offset) { - read_length = self->length - self->offset; + if (length > self->length - self->offset) { + length = self->length - self->offset; } - PyObject *buffer = inner_read(self, read_length, self->offset); + PyObject *buffer = inner_read(self, length, self->offset); if (buffer != NULL) { /* update offset on sucess. */ - self->offset += read_length; + self->offset += length; } return buffer; diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index fa995a4938389b..ef4b2b52ae8239 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -21,7 +21,7 @@ blob_close(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(blob_read__doc__, -"read($self, read_length=-1, /)\n" +"read($self, length=-1, /)\n" "--\n" "\n" "Read data from blob."); @@ -30,13 +30,13 @@ PyDoc_STRVAR(blob_read__doc__, {"read", (PyCFunction)(void(*)(void))blob_read, METH_FASTCALL, blob_read__doc__}, static PyObject * -blob_read_impl(pysqlite_Blob *self, int read_length); +blob_read_impl(pysqlite_Blob *self, int length); static PyObject * blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int read_length = -1; + int length = -1; if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { goto exit; @@ -44,12 +44,12 @@ blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - read_length = _PyLong_AsInt(args[0]); - if (read_length == -1 && PyErr_Occurred()) { + length = _PyLong_AsInt(args[0]); + if (length == -1 && PyErr_Occurred()) { goto exit; } skip_optional: - return_value = blob_read_impl(self, read_length); + return_value = blob_read_impl(self, length); exit: return return_value; @@ -199,4 +199,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=76c066429020440e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=755e33bbf7642839 input=a9049054013a1b77]*/ From 32132cf53b751147f333bebf3952c0dcbdbab276 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 14:28:16 +0200 Subject: [PATCH 30/77] Wrap error handling in support function --- Modules/_sqlite/blob.c | 65 ++++++++++++------------------------------ 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index a812f6f8fd3712..852e705eb58747 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -54,6 +54,20 @@ blob_dealloc(pysqlite_Blob *self) Py_DECREF(tp); } +static void +blob_seterror(pysqlite_Blob *self, int rc) +{ + assert(self->connection != NULL); +#if SQLITE_VERSION_NUMBER < 3008008 + // SQLite pre 3.8.8 does not set errors on the connection + if (rc == SQLITE_ABORT) { + PyErr_SetString(self->connection->OperationalError, + "Cannot operate on modified blob"); + return; + } +#endif + _pysqlite_seterror(self->connection->state, self->connection->db); +} /* * Checks if a blob object is usable (i. e. not closed). @@ -133,15 +147,7 @@ inner_read(pysqlite_Blob *self, int length, int offset) if (rc != SQLITE_OK) { Py_DECREF(buffer); - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, self->connection->db); - } + blob_seterror(self, rc); return NULL; } return buffer; @@ -194,15 +200,7 @@ write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, self->connection->db); - } + blob_seterror(self, rc); return -1; } return 0; @@ -490,16 +488,7 @@ blob_subscript(pysqlite_Blob *self, PyObject *item) Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, - self->connection->db); - } + blob_seterror(self, rc); PyMem_Free(result_buf); PyMem_Free(data_buff); return NULL; @@ -596,15 +585,7 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) Py_END_ALLOW_THREADS if (rc != SQLITE_OK){ - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, self->connection->db); - } + blob_seterror(self, rc); PyMem_Free(data_buff); rc = -1; } @@ -618,15 +599,7 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) Py_END_ALLOW_THREADS if (rc != SQLITE_OK){ - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, self->connection->db); - } + blob_seterror(self, rc); PyMem_Free(data_buff); rc = -1; } From 287803e68f130a255a3dea6f74de75e667518870 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 14:38:10 +0200 Subject: [PATCH 31/77] Add blob seek position markers as constants --- Lib/sqlite3/test/dbapi.py | 13 +++++++++---- Modules/_sqlite/blob.c | 10 ++++++---- Modules/_sqlite/blob.h | 4 ++++ Modules/_sqlite/module.c | 3 +++ 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 5615aee476ab0c..ffb6188bc10681 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -172,6 +172,11 @@ def test_module_constants(self): if sqlite.version_info >= (3, 8, 3): consts.append("SQLITE_RECURSIVE") consts += ["PARSE_DECLTYPES", "PARSE_COLNAMES"] + consts += [ + "BLOB_SEEK_START", + "BLOB_SEEK_CUR", + "BLOB_SEEK_END", + ] for const in consts: with self.subTest(const=const): self.assertTrue(hasattr(sqlite, const)) @@ -751,16 +756,16 @@ def test_blob_tell(self): def test_blob_seek_from_start(self): self.blob.seek(10) self.assertEqual(self.blob.tell(), 10) - self.blob.seek(10, 0) + self.blob.seek(10, sqlite.BLOB_SEEK_START) self.assertEqual(self.blob.tell(), 10) def test_blob_seek_from_current_pos(self): - self.blob.seek(10, 1) - self.blob.seek(10, 1) + self.blob.seek(10, sqlite.BLOB_SEEK_CUR) + self.blob.seek(10, sqlite.BLOB_SEEK_CUR) self.assertEqual(self.blob.tell(), 20) def test_blob_seek_from_end(self): - self.blob.seek(-10, 2) + self.blob.seek(-10, sqlite.BLOB_SEEK_END) self.assertEqual(self.blob.tell(), 90) def test_blob_seek_error(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 852e705eb58747..32941f966655b0 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -266,22 +266,24 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) } switch (origin) { - case 0: // relative to blob begin + case BLOB_SEEK_START: break; - case 1: // relative to current position + case BLOB_SEEK_CUR: if (offset > INT_MAX - self->offset) { goto overflow; } offset = self->offset + offset; break; - case 2: // relative to blob end + case BLOB_SEEK_END: if (offset > INT_MAX - self->length) { goto overflow; } offset = self->length + offset; break; default: - PyErr_SetString(PyExc_ValueError, "'origin' should be 0, 1 or 2"); + PyErr_SetString(PyExc_ValueError, + "'origin' should be 'BLOB_SEEK_START', " + "'BLOB_SEEK_CUR', or 'BLOB_SEEK_END'"); return NULL; } diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index cb8a5ebba191b7..9e768d80974b32 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -4,6 +4,10 @@ #include "sqlite3.h" #include "connection.h" +#define BLOB_SEEK_START 0 +#define BLOB_SEEK_CUR 1 +#define BLOB_SEEK_END 2 + typedef struct { PyObject_HEAD pysqlite_Connection *connection; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 2207cb4e086bf7..7865df99aa160f 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -393,6 +393,9 @@ static int add_integer_constants(PyObject *module) { #if SQLITE_VERSION_NUMBER >= 3008003 ret += PyModule_AddIntMacro(module, SQLITE_RECURSIVE); #endif + ret += PyModule_AddIntMacro(module, BLOB_SEEK_START); + ret += PyModule_AddIntMacro(module, BLOB_SEEK_CUR); + ret += PyModule_AddIntMacro(module, BLOB_SEEK_END); return ret; } From 0fc5a3948b7061a810282db06c9ef28c0688a114 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 21:17:25 +0200 Subject: [PATCH 32/77] Adjust SQLITE_ABORT error message --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 32941f966655b0..554966a18dba58 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -62,7 +62,7 @@ blob_seterror(pysqlite_Blob *self, int rc) // SQLite pre 3.8.8 does not set errors on the connection if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); + "Cannot operate on an expired blob handle"); return; } #endif From cd0bde10dad3c2bb2a1bc85d455f3a57f32f3de2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 22:48:58 +0200 Subject: [PATCH 33/77] Remove unneeded sqlite3_blob_close() and fix GC tracking --- Modules/_sqlite/connection.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 5587037fc6939f..958ade836dc630 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -424,6 +424,8 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, obj->length = len; obj->in_weakreflist = NULL; + PyObject_GC_Track(obj); + // Add our blob to connection blobs list PyObject *weakref = PyWeakref_NewRef((PyObject *)obj, NULL); if (weakref == NULL) { @@ -439,10 +441,6 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, error: Py_XDECREF(obj); - - Py_BEGIN_ALLOW_THREADS - sqlite3_blob_close(blob); - Py_END_ALLOW_THREADS return NULL; } From cf7e15edfc58bf209007bc51312bb1c8b34a3efe Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 22:53:11 +0200 Subject: [PATCH 34/77] Use close_blob() in pysqlite_close_all_blobs() --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 554966a18dba58..0a6f92467ba41c 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -117,7 +117,7 @@ pysqlite_close_all_blobs(pysqlite_Connection *self) PyObject *weakref = PyList_GET_ITEM(self->blobs, i); PyObject *blob = PyWeakref_GetObject(weakref); if (!Py_IsNone(blob)) { - blob_close_impl((pysqlite_Blob *)blob); + close_blob((pysqlite_Blob *)blob); } } } From 7e77217c4b0f4e151d8cb57358083dd8b8ba6e16 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 23:04:19 +0200 Subject: [PATCH 35/77] Refactor write/read --- Modules/_sqlite/blob.c | 35 ++++++++++++++------------------- Modules/_sqlite/clinic/blob.c.h | 20 +++++++++---------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 0a6f92467ba41c..b4dc07a8fbf242 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -182,11 +182,10 @@ blob_read_impl(pysqlite_Blob *self, int length) } PyObject *buffer = inner_read(self, length, self->offset); - if (buffer != NULL) { - /* update offset on sucess. */ - self->offset += length; + if (buffer == NULL) { + return NULL; } - + self->offset += length; return buffer; }; @@ -210,40 +209,36 @@ write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) /*[clinic input] _sqlite3.Blob.write as blob_write - data_buffer: Py_buffer + data: Py_buffer / Write data to blob. [clinic start generated code]*/ static PyObject * -blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer) -/*[clinic end generated code: output=dc8da6900b969799 input=8597402caf368add]*/ +blob_write_impl(pysqlite_Blob *self, Py_buffer *data) +/*[clinic end generated code: output=b34cf22601b570b2 input=0dcf4018286f55d2]*/ { - int rc; - - if (data_buffer->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); + if (!check_blob(self)) { return NULL; } - if (data_buffer->len > self->length - self->offset) { - PyErr_SetString(PyExc_ValueError, "data longer than blob length"); + if (data->len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); return NULL; } - if (!check_blob(self)) { + if (data->len > self->length - self->offset) { + PyErr_SetString(PyExc_ValueError, "data longer than blob length"); return NULL; } - rc = write_inner(self, data_buffer->buf, data_buffer->len, self->offset); - - if (rc == 0) { - self->offset += (int)data_buffer->len; - Py_RETURN_NONE; - } else { + int rc = write_inner(self, data->buf, data->len, self->offset); + if (rc < 0) { return NULL; } + self->offset += (int)data->len; + Py_RETURN_NONE; } diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index ef4b2b52ae8239..8276f8e140c2e8 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -56,7 +56,7 @@ blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(blob_write__doc__, -"write($self, data_buffer, /)\n" +"write($self, data, /)\n" "--\n" "\n" "Write data to blob."); @@ -65,27 +65,27 @@ PyDoc_STRVAR(blob_write__doc__, {"write", (PyCFunction)blob_write, METH_O, blob_write__doc__}, static PyObject * -blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer); +blob_write_impl(pysqlite_Blob *self, Py_buffer *data); static PyObject * blob_write(pysqlite_Blob *self, PyObject *arg) { PyObject *return_value = NULL; - Py_buffer data_buffer = {NULL, NULL}; + Py_buffer data = {NULL, NULL}; - if (PyObject_GetBuffer(arg, &data_buffer, PyBUF_SIMPLE) != 0) { + if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data_buffer, 'C')) { + if (!PyBuffer_IsContiguous(&data, 'C')) { _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); goto exit; } - return_value = blob_write_impl(self, &data_buffer); + return_value = blob_write_impl(self, &data); exit: - /* Cleanup for data_buffer */ - if (data_buffer.obj) { - PyBuffer_Release(&data_buffer); + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); } return return_value; @@ -199,4 +199,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=755e33bbf7642839 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=235d02d1bfa39b2a input=a9049054013a1b77]*/ From c57e45ae7c9d16d6c0cb5576c71e4cda669bf30a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 23:09:52 +0200 Subject: [PATCH 36/77] Sipmlify __exit__ --- Modules/_sqlite/blob.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index b4dc07a8fbf242..fd6d0753c2e327 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -346,17 +346,10 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, PyObject *tb) /*[clinic end generated code: output=fc86ceeb2b68c7b2 input=575d9ecea205f35f]*/ { - PyObject *res; if (!check_blob(self)) { return NULL; } - - res = blob_close_impl(self); - if (!res) { - return NULL; - } - Py_XDECREF(res); - + close_blob(self); Py_RETURN_FALSE; } From dd76f727f1b87a9b622734afeac8d2dbcc862865 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 23:45:22 +0200 Subject: [PATCH 37/77] Use new slice API --- Modules/_sqlite/blob.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index fd6d0753c2e327..d14d445918b951 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -448,11 +448,10 @@ blob_subscript(pysqlite_Blob *self, PyObject *item) } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - - if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, - &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); @@ -537,10 +536,10 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, - &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Blob object doesn't support slice deletion"); From 237684ed84111ab2b691175fb28ff71417e2e8b9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 21:54:09 +0200 Subject: [PATCH 38/77] Refactor very large functions --- Modules/_sqlite/blob.c | 293 ++++++++++++++++++++++------------------- 1 file changed, 156 insertions(+), 137 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index d14d445918b951..60a006f23a8020 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -423,186 +423,205 @@ blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) return write_inner(self, buf, 1, i); } +static PyObject * +subscript_index(pysqlite_Blob *self, PyObject *item) +{ + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) { + return NULL; + } + if (i < 0) { + i += self->length; + } + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return NULL; + } + // TODO: I am not sure... + return inner_read(self, 1, i); +} static PyObject * -blob_subscript(pysqlite_Blob *self, PyObject *item) +subscript_slice(pysqlite_Blob *self, PyObject *item) { - if (!check_blob(self)) { + Py_ssize_t start, stop, step, slicelen; + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) { - return NULL; - } - if (i < 0) { - i += self->length; - } - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return NULL; - } - // TODO: I am not sure... - return inner_read(self, 1, i); + if (slicelen <= 0) { + return PyBytes_FromStringAndSize("", 0); } - else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelen; - if (PySlice_Unpack(item, &start, &stop, &step) < 0) { - return NULL; + else if (step == 1) { + return inner_read(self, slicelen, start); + } + else { + char *result_buf = (char *)PyMem_Malloc(slicelen); + if (result_buf == NULL) { + return PyErr_NoMemory(); } - slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); - if (slicelen <= 0) { - return PyBytes_FromStringAndSize("", 0); - } - else if (step == 1) { - return inner_read(self, slicelen, start); + char *data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyMem_Free(result_buf); + return PyErr_NoMemory(); } - else { - char *result_buf = (char *)PyMem_Malloc(slicelen); - if (result_buf == NULL) { - return PyErr_NoMemory(); - } - - char *data_buff = (char *)PyMem_Malloc(stop - start); - if (data_buff == NULL) { - PyMem_Free(result_buf); - return PyErr_NoMemory(); - } - - int rc; - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS - if (rc != SQLITE_OK) { - blob_seterror(self, rc); - PyMem_Free(result_buf); - PyMem_Free(data_buff); - return NULL; - } + int rc; + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS - for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { - result_buf[i] = data_buff[cur]; - } - PyObject *result = PyBytes_FromStringAndSize(result_buf, slicelen); + if (rc != SQLITE_OK) { + blob_seterror(self, rc); PyMem_Free(result_buf); PyMem_Free(data_buff); - return result; + return NULL; + } + + for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { + result_buf[i] = data_buff[cur]; } + PyObject *result = PyBytes_FromStringAndSize(result_buf, slicelen); + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return result; } - else { - PyErr_SetString(PyExc_TypeError, "Blob indices must be integers"); +} + +static PyObject * +blob_subscript(pysqlite_Blob *self, PyObject *item) +{ + if (!check_blob(self)) { return NULL; } + + if (PyIndex_Check(item)) { + return subscript_index(self, item); + } + if (PySlice_Check(item)) { + return subscript_slice(self, item); + } + + PyErr_SetString(PyExc_TypeError, "Blob indices must be integers"); + return NULL; } +static int +ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) +{ + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) { + return -1; + } + if (i < 0) { + i += self->length; + } + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + const char *buf = PyBytes_AsString(value); + return write_inner(self, buf, 1, i); +} static int -blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) { int rc = -1; - if (!check_blob(self)) { + Py_ssize_t start, stop, step, slicelen; + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { + return -1; + } + slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support slice deletion"); return -1; } - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) { - return -1; - } - if (i < 0) { - i += self->length; - } - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return -1; - } - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob doesn't support item deletion"); - return -1; - } - if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { - PyErr_SetString(PyExc_IndexError, - "Blob assignment must be length-1 bytes()"); - return -1; - } + Py_buffer vbuf; + if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { + return -1; + } + if (vbuf.len != slicelen) { + PyErr_SetString(PyExc_IndexError, + "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; + } - const char *buf = PyBytes_AsString(value); - return write_inner(self, buf, 1, i); + if (slicelen == 0) { // FIXME } - else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelen; - if (PySlice_Unpack(item, &start, &stop, &step) < 0) { - return -1; - } - slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support slice deletion"); + else if (step == 1) { + rc = write_inner(self, vbuf.buf, slicelen, start); + } + else { + char *data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyErr_NoMemory(); return -1; } - Py_buffer vbuf; - if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { - return -1; - } - if (vbuf.len != slicelen) { - PyErr_SetString(PyExc_IndexError, - "Blob slice assignment is wrong size"); - PyBuffer_Release(&vbuf); - return -1; - } + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS - if (slicelen == 0) { // FIXME - } - else if (step == 1) { - rc = write_inner(self, vbuf.buf, slicelen, start); + if (rc != SQLITE_OK){ + blob_seterror(self, rc); + PyMem_Free(data_buff); + rc = -1; } - else { - char *data_buff = (char *)PyMem_Malloc(stop - start); - if (data_buff == NULL) { - PyErr_NoMemory(); - return -1; - } - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS + for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { + data_buff[cur] = ((char *)vbuf.buf)[i]; + } - if (rc != SQLITE_OK){ - blob_seterror(self, rc); - PyMem_Free(data_buff); - rc = -1; - } + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS - for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { - data_buff[cur] = ((char *)vbuf.buf)[i]; - } + if (rc != SQLITE_OK){ + blob_seterror(self, rc); + PyMem_Free(data_buff); + rc = -1; + } + rc = 0; - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS + } + PyBuffer_Release(&vbuf); + return rc; +} - if (rc != SQLITE_OK){ - blob_seterror(self, rc); - PyMem_Free(data_buff); - rc = -1; - } - rc = 0; +static int +blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +{ + if (!check_blob(self)) { + return -1; + } - } - PyBuffer_Release(&vbuf); - return rc; + if (PyIndex_Check(item)) { + return ass_subscript_index(self, item, value); } - else { - PyErr_SetString(PyExc_TypeError, - "Blob indices must be integer"); - return -1; + if (PySlice_Check(item)) { + return ass_subscript_slice(self, item, value); } + + PyErr_SetString(PyExc_TypeError, "Blob indices must be integer"); + return -1; } From 58905e81b86c17d7d8c0ab19b0fef8af7ec1dbe3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 22:06:20 +0200 Subject: [PATCH 39/77] Simplify subscript slice --- Modules/_sqlite/blob.c | 46 ++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 60a006f23a8020..60b917494ad454 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -138,8 +138,8 @@ inner_read(pysqlite_Blob *self, int length, int offset) if (buffer == NULL) { return NULL; } - char *raw_buffer = PyBytes_AS_STRING(buffer); + char *raw_buffer = PyBytes_AS_STRING(buffer); int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_read(self->blob, raw_buffer, length, self->offset); @@ -456,38 +456,26 @@ subscript_slice(pysqlite_Blob *self, PyObject *item) else if (step == 1) { return inner_read(self, slicelen, start); } - else { - char *result_buf = (char *)PyMem_Malloc(slicelen); - if (result_buf == NULL) { - return PyErr_NoMemory(); - } - char *data_buff = (char *)PyMem_Malloc(stop - start); - if (data_buff == NULL) { - PyMem_Free(result_buf); - return PyErr_NoMemory(); - } - - int rc; - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS + PyObject *blob = inner_read(self, stop - start, start); + if (blob == NULL) { + return NULL; + } - if (rc != SQLITE_OK) { - blob_seterror(self, rc); - PyMem_Free(result_buf); - PyMem_Free(data_buff); - return NULL; - } + PyObject *result = PyBytes_FromStringAndSize(NULL, slicelen); + if (result == NULL) { + goto exit; + } - for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { - result_buf[i] = data_buff[cur]; - } - PyObject *result = PyBytes_FromStringAndSize(result_buf, slicelen); - PyMem_Free(result_buf); - PyMem_Free(data_buff); - return result; + char *blob_buf = PyBytes_AS_STRING(blob); + char *res_buf = PyBytes_AS_STRING(result); + for (Py_ssize_t i = 0, j = 0; i < slicelen; i++, j += step) { + res_buf[i] = blob_buf[j]; } + +exit: + Py_DECREF(blob); + return result; } static PyObject * From 9d69fcade2f56c6f00c7dd2d9cb99145dd80c384 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 22:24:47 +0200 Subject: [PATCH 40/77] Consolidate more tests --- Lib/sqlite3/test/dbapi.py | 55 +++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index ffb6188bc10681..d752b2c76dd6ae 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -750,21 +750,16 @@ def tearDown(self): def test_blob_length(self): self.assertEqual(len(self.blob), 100) - def test_blob_tell(self): - self.assertEqual(self.blob.tell(), 0) - - def test_blob_seek_from_start(self): + def test_blob_seek_and_tell(self): self.blob.seek(10) self.assertEqual(self.blob.tell(), 10) + self.blob.seek(10, sqlite.BLOB_SEEK_START) self.assertEqual(self.blob.tell(), 10) - def test_blob_seek_from_current_pos(self): - self.blob.seek(10, sqlite.BLOB_SEEK_CUR) self.blob.seek(10, sqlite.BLOB_SEEK_CUR) self.assertEqual(self.blob.tell(), 20) - def test_blob_seek_from_end(self): self.blob.seek(-10, sqlite.BLOB_SEEK_END) self.assertEqual(self.blob.tell(), 90) @@ -824,37 +819,35 @@ def test_blob_write_when_readonly(self): read_only_blob.write(b"aaa") read_only_blob.close() - def test_blob_open_with_bad_db(self): - with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("test", "blob_col", 1, name="notexisintg") - - def test_blob_open_with_bad_table(self): - with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("notexisintg", "blob_col", 1) - - def test_blob_open_with_bad_column(self): - with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("test", "notexisting", 1) - - def test_blob_open_with_bad_row(self): - with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("test", "blob_col", 2) + def test_blob_open_error(self): + dataset = ( + (("test", "blob_col", 1), {"name": "notexisting"}), + (("notexisting", "blob_col", 1), {}), + (("test", "notexisting", 1), {}), + (("test", "blob_col", 2), {}), + ) + for args, kwds in dataset: + with self.subTest(args=args, kwds=kwds): + with self.assertRaises(sqlite.OperationalError): + self.cx.open_blob(*args, **kwds) def test_blob_get_item(self): self.assertEqual(self.blob[5], b"a") - def test_blob_get_item_index_out_of_range(self): - with self.assertRaises(IndexError): - self.blob[105] - with self.assertRaises(IndexError): - self.blob[-105] - def test_blob_get_item_negative_index(self): self.assertEqual(self.blob[-5], b"a") - def test_blob_get_item_invalid_index(self): - with self.assertRaises(TypeError): - self.blob[b"a"] + def test_blob_get_item_error(self): + dataset = ( + (b"", TypeError), + (105, IndexError), + (-105, IndexError), + (len(self.blob), IndexError), + ) + for idx, exc in dataset: + with self.subTest(idx=idx, exc=exc): + with self.assertRaises(exc): + self.blob[idx] def test_blob_get_slice(self): self.assertEqual(self.blob[5:10], b"aaaaa") From 285bb3d2d57bd10a12e57e22ff67cf86d009f8f6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 22:39:04 +0200 Subject: [PATCH 41/77] Use supplied offset in inner_read() --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 60b917494ad454..6916380cbfba3b 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -142,7 +142,7 @@ inner_read(pysqlite_Blob *self, int length, int offset) char *raw_buffer = PyBytes_AS_STRING(buffer); int rc; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, raw_buffer, length, self->offset); + rc = sqlite3_blob_read(self->blob, raw_buffer, length, offset); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { From 2f3051a6705f76d0039fa44ec36ec7f6168cacaa Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 22:45:50 +0200 Subject: [PATCH 42/77] Simplify test --- Lib/sqlite3/test/dbapi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index d752b2c76dd6ae..9d1eae194076a9 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -737,9 +737,9 @@ def test_same_query_in_multiple_cursors(self): class BlobTests(unittest.TestCase): def setUp(self): self.cx = sqlite.connect(":memory:") - self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("create table test(blob_col blob)") self.blob_data = b"a" * 100 - self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data, )) + self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data,)) self.blob = self.cx.open_blob("test", "blob_col", 1) self.second_data = b"b" * 100 @@ -803,12 +803,12 @@ def test_blob_write_more_then_blob_size(self): self.blob.write(b"a" * 1000) def test_blob_read_after_row_change(self): - self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + self.cx.execute("UPDATE test SET blob_col='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.read() def test_blob_write_after_row_change(self): - self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + self.cx.execute("UPDATE test SET blob_col='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") @@ -920,7 +920,7 @@ def test_blob_closed(self): def test_closed_blob_read(self): con = sqlite.connect(":memory:") - con.execute("create table test(id integer primary key, blob_col blob)") + con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(100))") blob = con.open_blob("test", "blob_col", 1) con.close() From fd7c31173b1ec3d5dd6242ad9f1a12783ef5dfec Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 23:13:03 +0200 Subject: [PATCH 43/77] Simplify assign subscript slice --- Modules/_sqlite/blob.c | 52 ++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 6916380cbfba3b..9394dbe3bea5f4 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -528,18 +528,17 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) static int ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) { - int rc = -1; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support slice deletion"); + return -1; + } Py_ssize_t start, stop, step, slicelen; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support slice deletion"); - return -1; - } Py_buffer vbuf; if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { @@ -552,43 +551,26 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } - if (slicelen == 0) { // FIXME + int rc; + if (slicelen == 0) { + rc = 0; } else if (step == 1) { rc = write_inner(self, vbuf.buf, slicelen, start); } else { - char *data_buff = (char *)PyMem_Malloc(stop - start); - if (data_buff == NULL) { - PyErr_NoMemory(); - return -1; - } - - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS - - if (rc != SQLITE_OK){ - blob_seterror(self, rc); - PyMem_Free(data_buff); + PyObject *read_blob = inner_read(self, stop - start, start); + if (read_blob == NULL) { rc = -1; } - - for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { - data_buff[cur] = ((char *)vbuf.buf)[i]; - } - - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS - - if (rc != SQLITE_OK){ - blob_seterror(self, rc); - PyMem_Free(data_buff); - rc = -1; + else { + char *blob_buf = PyBytes_AS_STRING(read_blob); + for (Py_ssize_t i = 0, j = 0; i < slicelen; i++, j += step) { + blob_buf[j] = ((char *)vbuf.buf)[i]; + } + rc = write_inner(self, blob_buf, stop - start, start); + Py_DECREF(read_blob); } - rc = 0; - } PyBuffer_Release(&vbuf); return rc; From 0644e8794a51147304d6d72cfac8bc0e6f25e9d6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 23:21:58 +0200 Subject: [PATCH 44/77] Early error checking, and use PyBytes_AS_STRING() when possible --- Modules/_sqlite/blob.c | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 9394dbe3bea5f4..5e69812e45ceb5 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -398,28 +398,29 @@ blob_item(pysqlite_Blob *self, Py_ssize_t i) } static int -blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) +blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *value) { if (!check_blob(self)) { return -1; } - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return -1; - } - if (v == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Blob object doesn't support item deletion"); return -1; } - if (! (PyBytes_Check(v) && PyBytes_Size(v)==1) ) { + if (!PyBytes_Check(value) || PyBytes_Size(value) != 1) { PyErr_SetString(PyExc_IndexError, "Blob assignment must be length-1 bytes()"); return -1; } - const char *buf = PyBytes_AsString(v); + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + + const char *buf = PyBytes_AS_STRING(value); return write_inner(self, buf, 1, i); } @@ -499,6 +500,17 @@ blob_subscript(pysqlite_Blob *self, PyObject *item) static int ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) { + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob doesn't support item deletion"); + return -1; + } + if (!PyBytes_Check(value) || PyBytes_Size(value) != 1) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) { return -1; @@ -510,18 +522,8 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob doesn't support item deletion"); - return -1; - } - if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { - PyErr_SetString(PyExc_IndexError, - "Blob assignment must be length-1 bytes()"); - return -1; - } - const char *buf = PyBytes_AsString(value); + const char *buf = PyBytes_AS_STRING(value); return write_inner(self, buf, 1, i); } From ced431a7873a871d761f4274e0594ebd860b17b8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 23:24:54 +0200 Subject: [PATCH 45/77] Expand test suite --- Lib/sqlite3/test/dbapi.py | 50 +++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 9d1eae194076a9..119272e8429b4a 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -764,15 +764,24 @@ def test_blob_seek_and_tell(self): self.assertEqual(self.blob.tell(), 90) def test_blob_seek_error(self): - for pos in 1000, -10: - with self.subTest(pos=pos): - self.assertRaises(ValueError, self.blob.seek, pos) + ops = ( + lambda: self.blob.seek(1000), + lambda: self.blob.seek(-10), + lambda: self.blob.seek(10, -1), + ) + for op in ops: + with self.subTest(op=op): + self.assertRaises(ValueError, op) def test_blob_read(self): - self.assertEqual(self.blob.read(), self.blob_data) + buf = self.blob.read() + self.assertEqual(buf, self.blob_data) + self.assertEqual(len(buf), len(self.blob_data)) - def test_blob_read_size(self): - self.assertEqual(len(self.blob.read(10)), 10) + def test_blob_read_too_much(self): + buf = self.blob.read(len(self.blob_data) * 2) + self.assertEqual(buf, self.blob_data) + self.assertEqual(len(buf), len(self.blob_data)) def test_blob_read_advance_offset(self): self.blob.read(10) @@ -867,10 +876,25 @@ def test_blob_set_item(self): self.blob[0] = b"b" self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"b" + self.blob_data[1:]) + def test_blob_set_item(self): + self.blob[-1] = b"z" + self.assertEqual(self.blob[-1], b"z") + + def test_blob_set_item_error(self): + with self.assertRaises(TypeError): + self.blob["a"] = b"b" + with self.assertRaises(TypeError): + del self.blob[0] + with self.assertRaises(IndexError): + self.blob[1000] = b"a" + def test_blob_set_slice(self): self.blob[0:5] = b"bbbbb" self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bbbbb" + self.blob_data[5:]) + def test_blob_set_empty_slice(self): + self.blob[0:0] = b"" + def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"bbbbb" self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bababababa" + self.blob_data[10:]) @@ -878,9 +902,13 @@ def test_blob_set_slice_with_skip(self): def test_blob_get_empty_slice(self): self.assertEqual(self.blob[5:5], b"") - def test_blob_set_slice_wrong_length(self): + def test_blob_set_slice_error(self): with self.assertRaises(IndexError): self.blob[5:10] = b"a" + with self.assertRaises(IndexError): + self.blob[5:10] = b"a" * 1000 + with self.assertRaises(TypeError): + del self.blob[5:10] def test_blob_concat_not_supported(self): with self.assertRaises(SystemError): @@ -906,11 +934,19 @@ def test_blob_closed(self): cx.execute("insert into test values (zeroblob(100))") blob = cx.open_blob("test", "b", 1) blob.close() + + def assign(): blob[0] = b"" ops = [ lambda: blob.read(), lambda: blob.write(b""), lambda: blob.seek(0), lambda: blob.tell(), + lambda: blob.__enter__(), + lambda: blob.__exit__(None, None, None), + lambda: len(blob), + lambda: blob[0], + lambda: blob[0:1], + assign, ] msg = "Cannot operate on a closed blob" for op in ops: From 2dae1b9efabeaed98d3d3c765290c8d3a2b32a96 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 23:40:34 +0200 Subject: [PATCH 46/77] Remove unneeded parts of sequence protocol --- Lib/sqlite3/test/dbapi.py | 20 +++++----- Modules/_sqlite/blob.c | 80 --------------------------------------- 2 files changed, 9 insertions(+), 91 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 119272e8429b4a..c947bdb222251a 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -910,17 +910,15 @@ def test_blob_set_slice_error(self): with self.assertRaises(TypeError): del self.blob[5:10] - def test_blob_concat_not_supported(self): - with self.assertRaises(SystemError): - self.blob + self.blob - - def test_blob_repeat_not_supported(self): - with self.assertRaises(SystemError): - self.blob * 5 - - def test_blob_contains_not_supported(self): - with self.assertRaises(SystemError): - b"aaaaa" in self.blob + def test_blob_sequence_not_supported(self): + ops = ( + lambda: self.blob + self.blob, + lambda: self.blob * 5, + lambda: b"a" in self.blob, + ) + for op in ops: + with self.subTest(op=op): + self.assertRaises(TypeError, op) def test_blob_context_manager(self): data = b"a" * 100 diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 5e69812e45ceb5..4c01a1fb63e690 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -353,77 +353,6 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, Py_RETURN_FALSE; } -static PyObject * -blob_concat(pysqlite_Blob *self, PyObject *args) -{ - if (check_blob(self)) { - PyErr_SetString(PyExc_SystemError, "Blob don't support concatenation"); - } - return NULL; -} - -static PyObject * -blob_repeat(pysqlite_Blob *self, PyObject *args) -{ - if (check_blob(self)) { - PyErr_SetString(PyExc_SystemError, - "Blob don't support repeat operation"); - } - return NULL; -} - -static int -blob_contains(pysqlite_Blob *self, PyObject *args) -{ - if (check_blob(self)) { - PyErr_SetString(PyExc_SystemError, - "Blob don't support contains operation"); - } - return -1; -} - -static PyObject * -blob_item(pysqlite_Blob *self, Py_ssize_t i) -{ - if (!check_blob(self)) { - return NULL; - } - - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return NULL; - } - - return inner_read(self, 1, i); -} - -static int -blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *value) -{ - if (!check_blob(self)) { - return -1; - } - - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support item deletion"); - return -1; - } - if (!PyBytes_Check(value) || PyBytes_Size(value) != 1) { - PyErr_SetString(PyExc_IndexError, - "Blob assignment must be length-1 bytes()"); - return -1; - } - - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return -1; - } - - const char *buf = PyBytes_AS_STRING(value); - return write_inner(self, buf, 1, i); -} - static PyObject * subscript_index(pysqlite_Blob *self, PyObject *item) { @@ -438,7 +367,6 @@ subscript_index(pysqlite_Blob *self, PyObject *item) PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } - // TODO: I am not sure... return inner_read(self, 1, i); } @@ -619,14 +547,6 @@ static PyType_Slot blob_slots[] = { {Py_tp_methods, blob_methods}, {Py_tp_members, blob_members}, - // Sequence protocol - {Py_sq_length, blob_length}, - {Py_sq_concat, blob_concat}, - {Py_sq_repeat, blob_repeat}, - {Py_sq_item, blob_item}, - {Py_sq_ass_item, blob_ass_item}, - {Py_sq_contains, blob_contains}, - // Mapping protocol {Py_mp_length, blob_length}, {Py_mp_subscript, blob_subscript}, From e8fa47e5665709a4c8f998ce5343116a26f3ba89 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:12:46 +0200 Subject: [PATCH 47/77] Normalise error messages --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 4c01a1fb63e690..1161da229e75af 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -460,7 +460,7 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) { if (value == NULL) { PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support slice deletion"); + "Blob doesn't support slice deletion"); return -1; } From 644166845a44f79052b2ec4955f0238a284f70e1 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:18:49 +0200 Subject: [PATCH 48/77] Improve error message/type for to large subscript assignment --- Lib/sqlite3/test/dbapi.py | 2 ++ Modules/_sqlite/blob.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index c947bdb222251a..44f44d85a6b67c 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -883,6 +883,8 @@ def test_blob_set_item(self): def test_blob_set_item_error(self): with self.assertRaises(TypeError): self.blob["a"] = b"b" + with self.assertRaises(ValueError): + self.blob[0] = b"abc" with self.assertRaises(TypeError): del self.blob[0] with self.assertRaises(IndexError): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 1161da229e75af..f8e879acd156a3 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -434,8 +434,8 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } if (!PyBytes_Check(value) || PyBytes_Size(value) != 1) { - PyErr_SetString(PyExc_IndexError, - "Blob assignment must be length-1 bytes()"); + PyErr_SetString(PyExc_ValueError, + "Blob assignment must be a single byte"); return -1; } From 03d2152dbcf96966abe0529c140f223c66232b41 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:28:03 +0200 Subject: [PATCH 49/77] Normalise naming: write_inner => inner_write --- Modules/_sqlite/blob.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f8e879acd156a3..a4a524d2edfb0d 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -190,7 +190,7 @@ blob_read_impl(pysqlite_Blob *self, int length) }; static int -write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) { int rc; @@ -233,7 +233,7 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) return NULL; } - int rc = write_inner(self, data->buf, data->len, self->offset); + int rc = inner_write(self, data->buf, data->len, self->offset); if (rc < 0) { return NULL; } @@ -452,7 +452,7 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) } const char *buf = PyBytes_AS_STRING(value); - return write_inner(self, buf, 1, i); + return inner_write(self, buf, 1, i); } static int @@ -486,7 +486,7 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) rc = 0; } else if (step == 1) { - rc = write_inner(self, vbuf.buf, slicelen, start); + rc = inner_write(self, vbuf.buf, slicelen, start); } else { PyObject *read_blob = inner_read(self, stop - start, start); @@ -498,7 +498,7 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) for (Py_ssize_t i = 0, j = 0; i < slicelen; i++, j += step) { blob_buf[j] = ((char *)vbuf.buf)[i]; } - rc = write_inner(self, blob_buf, stop - start, start); + rc = inner_write(self, blob_buf, stop - start, start); Py_DECREF(read_blob); } } From 9360e62cf3bba7a8a20f2a6948ce748f901be2ec Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:29:09 +0200 Subject: [PATCH 50/77] Adjust comment --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index a4a524d2edfb0d..92c38d3754aad2 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -59,7 +59,7 @@ blob_seterror(pysqlite_Blob *self, int rc) { assert(self->connection != NULL); #if SQLITE_VERSION_NUMBER < 3008008 - // SQLite pre 3.8.8 does not set errors on the connection + // SQLite pre 3.8.8 does not set this blob error on the connection if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on an expired blob handle"); From 8eb16f7905eb07ac34c1df52d85b1b0a597affaa Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:30:43 +0200 Subject: [PATCH 51/77] Move blob_seterror() closer to where it's first used --- Modules/_sqlite/blob.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 92c38d3754aad2..6bc0a3ba3923ae 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -54,21 +54,6 @@ blob_dealloc(pysqlite_Blob *self) Py_DECREF(tp); } -static void -blob_seterror(pysqlite_Blob *self, int rc) -{ - assert(self->connection != NULL); -#if SQLITE_VERSION_NUMBER < 3008008 - // SQLite pre 3.8.8 does not set this blob error on the connection - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on an expired blob handle"); - return; - } -#endif - _pysqlite_seterror(self->connection->state, self->connection->db); -} - /* * Checks if a blob object is usable (i. e. not closed). * @@ -131,6 +116,21 @@ blob_length(pysqlite_Blob *self) return self->length; }; +static void +blob_seterror(pysqlite_Blob *self, int rc) +{ + assert(self->connection != NULL); +#if SQLITE_VERSION_NUMBER < 3008008 + // SQLite pre 3.8.8 does not set this blob error on the connection + if (rc == SQLITE_ABORT) { + PyErr_SetString(self->connection->OperationalError, + "Cannot operate on an expired blob handle"); + return; + } +#endif + _pysqlite_seterror(self->connection->state, self->connection->db); +} + static PyObject * inner_read(pysqlite_Blob *self, int length, int offset) { From 411d07e6e9ebd5779d9d49f7b3a0b31ba9e49616 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:32:24 +0200 Subject: [PATCH 52/77] Fetch state from connection in check_blob() --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 6bc0a3ba3923ae..f8b8bc2938bbc7 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -67,7 +67,7 @@ check_blob(pysqlite_Blob *self) return 0; } if (self->blob == NULL) { - pysqlite_state *state = pysqlite_get_state(NULL); + pysqlite_state *state = self->connection->state; PyErr_SetString(state->ProgrammingError, "Cannot operate on a closed blob."); return 0; From 8149e36f642f6a5fb6e56187eea6a5d7dbb02bb3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:33:04 +0200 Subject: [PATCH 53/77] Remove unused declaration --- Modules/_sqlite/blob.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 9e768d80974b32..737df70ff205da 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -1,5 +1,6 @@ #ifndef PYSQLITE_BLOB_H #define PYSQLITE_BLOB_H + #include "Python.h" #include "sqlite3.h" #include "connection.h" @@ -18,8 +19,6 @@ typedef struct { PyObject *in_weakreflist; } pysqlite_Blob; -PyObject *pysqlite_blob_close(pysqlite_Blob *self); - int pysqlite_blob_setup_types(PyObject *module); void pysqlite_close_all_blobs(pysqlite_Connection *self); From 56b4caabc8746233b7d717beca7e66aa613d5ce3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:53:18 +0200 Subject: [PATCH 54/77] Doc adjustments --- Doc/includes/sqlite3/blob.py | 7 +++-- Doc/includes/sqlite3/blob_with.py | 6 ++--- Doc/library/sqlite3.rst | 45 ++++++++++++++++--------------- Lib/sqlite3/test/dbapi.py | 12 +++------ Modules/_sqlite/blob.c | 9 +++---- Modules/_sqlite/module.c | 3 --- 6 files changed, 37 insertions(+), 45 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index afd7812a8b3af9..f928349fd0e981 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -1,13 +1,12 @@ import sqlite3 con = sqlite3.connect(":memory:") -# creating the table -con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") -# opening blob handle + blob = con.open_blob("test", "blob_col", 1) blob.write(b"Hello") blob.write(b"World") blob.seek(0) -print(blob.read()) # will print b"HelloWorld" +print(blob.read()) # will print b"HelloWorld" blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index fdca9fbc638ea2..2e48d568830820 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -1,12 +1,12 @@ import sqlite3 con = sqlite3.connect(":memory:") -# creating the table + con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") -# opening blob handle + with con.open_blob("test", "blob_col", 1) as blob: blob.write(b"Hello") blob.write(b"World") blob.seek(0) - print(blob.read()) # will print b"HelloWorld" + print(blob.read()) # will print b"HelloWorld" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 4c6b24ec4ea5ce..ad621c5ae70fd3 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -304,20 +304,20 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. - .. method:: open_blob(table, column, row, *, readonly=False, dbname="main") + .. method:: open_blob(table, column, row, /, *, readonly=False, name="main") On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, - column *column*, table *table* in database *dbname* will be returned. - When *readonly* is :const:`True` the BLOB is opened with read - permissions. Otherwise the BLOB has read and write permissions. + column *column*, table *table* in database *name* will be returned. + When *readonly* is :const:`True` the BLOB is opened without write + permissions. .. note:: The BLOB size cannot be changed using the :class:`Blob` class. Use - ``zeroblob`` to create the blob in the wanted size in advance. + the SQL function ``zeroblob`` to create a blob with a fixed size. - .. versionadded:: 3.10 + .. versionadded:: 3.11 .. method:: commit() @@ -900,21 +900,21 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.10 +.. versionadded:: 3.11 .. class:: Blob A :class:`Blob` instance can read and write the data in the - :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both - the file and sequence protocol. For example, you can read data from the + :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` class implements + the file and mapping protocols. For example, you can read data from the :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. You can call ``len(obj)`` to get size of the BLOB. .. method:: Blob.close() - Close the BLOB now (rather than whenever __del__ is called). + Close the BLOB. - The BLOB will be unusable from this point forward; an + The BLOB will be unusable from this point forward. An :class:`~sqlite3.Error` (or subclass) exception will be raised if any operation is attempted with the BLOB. @@ -926,25 +926,26 @@ Blob Objects Read *size* bytes of data from the BLOB at the current offset position. If the end of the BLOB is reached we will return the data up to end of - file. When *size* is not specified or negative we will read up to end - of BLOB. + file. When *size* is not specified or is negative, :meth:`~Blob.read` + will read till the end of the BLOB. - .. method:: Blob.write(data) + .. method:: Blob.write(data, /) Write *data* to the BLOB at the current offset. This function cannot - changed BLOB length. If data write will result in writing to more - then BLOB current size an error will be raised. + change the BLOB length. Writing beyond the end of the blob will result in + an exception being raised. .. method:: Blob.tell() - Return the current offset of the BLOB. + Return the current access position of the BLOB. - .. method:: Blob.seek(offset, whence=os.SEEK_SET) + .. method:: Blob.seek(offset, /, origin=sqlite3.BLOB_SEEK_START) - Set the BLOB offset. The *whence* argument is optional and defaults to - :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values - are :data:`os.SEEK_CUR` or 1 (seek relative to the current position) and - :data:`os.SEEK_END` or 2 (seek relative to the BLOB’s end). + Set the current access position of the BLOB. The *origin* argument is + optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB + positioning); other values are :data:`os.SEEK_CUR` or 1 (seek relative to + the current position) and :data:`os.SEEK_END` or 2 (seek relative to the + BLOB’s end). :class:`Blob` example: diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 44f44d85a6b67c..183479f18ad341 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -172,11 +172,6 @@ def test_module_constants(self): if sqlite.version_info >= (3, 8, 3): consts.append("SQLITE_RECURSIVE") consts += ["PARSE_DECLTYPES", "PARSE_COLNAMES"] - consts += [ - "BLOB_SEEK_START", - "BLOB_SEEK_CUR", - "BLOB_SEEK_END", - ] for const in consts: with self.subTest(const=const): self.assertTrue(hasattr(sqlite, const)) @@ -754,13 +749,14 @@ def test_blob_seek_and_tell(self): self.blob.seek(10) self.assertEqual(self.blob.tell(), 10) - self.blob.seek(10, sqlite.BLOB_SEEK_START) + import os + self.blob.seek(10, os.SEEK_SET) self.assertEqual(self.blob.tell(), 10) - self.blob.seek(10, sqlite.BLOB_SEEK_CUR) + self.blob.seek(10, os.SEEK_CUR) self.assertEqual(self.blob.tell(), 20) - self.blob.seek(-10, sqlite.BLOB_SEEK_END) + self.blob.seek(-10, os.SEEK_END) self.assertEqual(self.blob.tell(), 90) def test_blob_seek_error(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f8b8bc2938bbc7..9c840e90b97eea 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -261,15 +261,15 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) } switch (origin) { - case BLOB_SEEK_START: + case 0: break; - case BLOB_SEEK_CUR: + case 1: if (offset > INT_MAX - self->offset) { goto overflow; } offset = self->offset + offset; break; - case BLOB_SEEK_END: + case 2: if (offset > INT_MAX - self->length) { goto overflow; } @@ -277,8 +277,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) break; default: PyErr_SetString(PyExc_ValueError, - "'origin' should be 'BLOB_SEEK_START', " - "'BLOB_SEEK_CUR', or 'BLOB_SEEK_END'"); + "'origin' should be 0, 1, or 2"); return NULL; } diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 7865df99aa160f..2207cb4e086bf7 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -393,9 +393,6 @@ static int add_integer_constants(PyObject *module) { #if SQLITE_VERSION_NUMBER >= 3008003 ret += PyModule_AddIntMacro(module, SQLITE_RECURSIVE); #endif - ret += PyModule_AddIntMacro(module, BLOB_SEEK_START); - ret += PyModule_AddIntMacro(module, BLOB_SEEK_CUR); - ret += PyModule_AddIntMacro(module, BLOB_SEEK_END); return ret; } From afdeb2e601eb7b646ece4413e363787ba5ce5a60 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:59:22 +0200 Subject: [PATCH 55/77] Add What's New and adjust NEWS --- Doc/whatsnew/3.11.rst | 4 ++++ .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 7 +++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 9befe8f2732e70..f3766efefc58f5 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -232,6 +232,10 @@ sqlite3 (Contributed by Aviv Palivoda, Daniel Shahaf, and Erlend E. Aasland in :issue:`16379`.) +* Add :meth:`~sqlite3.Connection.open_blob` to :class:`sqlite3.Connection`. + :class:`sqlite3.Blob` allows incremental I/O operations on blobs. + (Contributed by Aviv Palivoda and Erlend E. Aasland in :issue:`24905`) + Removed ======= diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst index c7d2405d89539e..c9571f374862f0 100644 --- a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -1,4 +1,3 @@ -The :class:`sqlite3.Connection` now has the -:meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` -allows incremental I/O operations to blobs. (Patch by Aviv Palivoda in -:issue:`24905`) +Add :meth:`~sqlite3.Connection.open_blob` to :class:`sqlite3.Connection`. +:class:`sqlite3.Blob` allows incremental I/O operations on blobs. +Patch by Aviv Palivoda and Erlend E. Aasland. From b12219fca30def125364aa1280c4b23ed88a023e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 21:52:22 +0200 Subject: [PATCH 56/77] Use sqlite3_blob_bytes() iso. storing length on blob object Prepare for adding reopen() support --- Modules/_sqlite/blob.c | 44 +++++++++++++++++++++++------------------- Modules/_sqlite/blob.h | 1 - 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 9c840e90b97eea..c67e17c7090c94 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -113,7 +113,7 @@ blob_length(pysqlite_Blob *self) if (!check_blob(self)) { return -1; } - return self->length; + return sqlite3_blob_bytes(self->blob); }; static void @@ -171,14 +171,12 @@ blob_read_impl(pysqlite_Blob *self, int length) return NULL; } - if (length < 0) { - /* same as file read. */ - length = self->length; - } - - /* making sure we don't read more then blob size */ - if (length > self->length - self->offset) { - length = self->length - self->offset; + /* Make sure we never read past "EOB". Also read the rest of the blob if a + * negative length is specified. */ + int blob_len = sqlite3_blob_bytes(self->blob); + int max_read_len = blob_len - self->offset; + if (length < 0 || length > max_read_len) { + length = max_read_len; } PyObject *buffer = inner_read(self, length, self->offset); @@ -228,7 +226,8 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) return NULL; } - if (data->len > self->length - self->offset) { + int remaining_len = sqlite3_blob_bytes(self->blob) - self->offset; + if (data->len > remaining_len) { PyErr_SetString(PyExc_ValueError, "data longer than blob length"); return NULL; } @@ -260,6 +259,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) return NULL; } + int blob_len = sqlite3_blob_bytes(self->blob); switch (origin) { case 0: break; @@ -267,13 +267,13 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) if (offset > INT_MAX - self->offset) { goto overflow; } - offset = self->offset + offset; + offset += self->offset; break; case 2: - if (offset > INT_MAX - self->length) { + if (offset > INT_MAX - blob_len) { goto overflow; } - offset = self->length + offset; + offset += blob_len; break; default: PyErr_SetString(PyExc_ValueError, @@ -281,7 +281,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) return NULL; } - if (offset < 0 || offset > self->length) { + if (offset < 0 || offset > blob_len) { PyErr_SetString(PyExc_ValueError, "offset out of blob range"); return NULL; } @@ -359,10 +359,11 @@ subscript_index(pysqlite_Blob *self, PyObject *item) if (i == -1 && PyErr_Occurred()) { return NULL; } + int blob_len = sqlite3_blob_bytes(self->blob); if (i < 0) { - i += self->length; + i += blob_len; } - if (i < 0 || i >= self->length) { + if (i < 0 || i >= blob_len) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } @@ -376,7 +377,8 @@ subscript_slice(pysqlite_Blob *self, PyObject *item) if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } - slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); + int blob_len = sqlite3_blob_bytes(self->blob); + slicelen = PySlice_AdjustIndices(blob_len, &start, &stop, step); if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); @@ -442,10 +444,11 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) if (i == -1 && PyErr_Occurred()) { return -1; } + int blob_len = sqlite3_blob_bytes(self->blob); if (i < 0) { - i += self->length; + i += blob_len; } - if (i < 0 || i >= self->length) { + if (i < 0 || i >= blob_len) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } @@ -467,7 +470,8 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } - slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); + int blob_len = sqlite3_blob_bytes(self->blob); + slicelen = PySlice_AdjustIndices(blob_len, &start, &stop, step); Py_buffer vbuf; if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 737df70ff205da..b4ac4ae0e6c7dc 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -14,7 +14,6 @@ typedef struct { pysqlite_Connection *connection; sqlite3_blob *blob; int offset; - int length; PyObject *in_weakreflist; } pysqlite_Blob; From 36b0ca196f53a50f59a0392c7cba6d157636503a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 21:57:17 +0200 Subject: [PATCH 57/77] Also remove length from blob_open() --- Modules/_sqlite/connection.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 45276ef73f46bd..e9ce33fa19b2bc 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -431,14 +431,11 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } - int rc, len; + int rc; sqlite3_blob *blob; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_open(self->db, name, table, col, row, !readonly, &blob); - if (rc == SQLITE_OK) { - len = sqlite3_blob_bytes(blob); - } Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -454,7 +451,6 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, obj->connection = (pysqlite_Connection *)Py_NewRef(self); obj->blob = blob; obj->offset = 0; - obj->length = len; obj->in_weakreflist = NULL; PyObject_GC_Track(obj); From 3d91705f3bc2cc5154e5d4f437dd1c805c2a59b6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 22:07:08 +0200 Subject: [PATCH 58/77] Add get subscript index helper --- Modules/_sqlite/blob.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index c67e17c7090c94..8dd3678c9f0ef7 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -352,12 +352,12 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, Py_RETURN_FALSE; } -static PyObject * -subscript_index(pysqlite_Blob *self, PyObject *item) +static int +get_subscript_index(pysqlite_Blob *self, PyObject *item) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) { - return NULL; + return -1; } int blob_len = sqlite3_blob_bytes(self->blob); if (i < 0) { @@ -365,6 +365,16 @@ subscript_index(pysqlite_Blob *self, PyObject *item) } if (i < 0 || i >= blob_len) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + return i; +} + +static PyObject * +subscript_index(pysqlite_Blob *self, PyObject *item) +{ + int i = get_subscript_index(self, item); + if (i < 0) { return NULL; } return inner_read(self, 1, i); @@ -439,20 +449,10 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) "Blob assignment must be a single byte"); return -1; } - - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) { - return -1; - } - int blob_len = sqlite3_blob_bytes(self->blob); + int i = get_subscript_index(self, item); if (i < 0) { - i += blob_len; - } - if (i < 0 || i >= blob_len) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } - const char *buf = PyBytes_AS_STRING(value); return inner_write(self, buf, 1, i); } From ceee315b56684644ec6f02cdca30dde1ed44a13d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 22:13:06 +0200 Subject: [PATCH 59/77] Expand test suite --- Lib/sqlite3/test/dbapi.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 183479f18ad341..aab7044955e762 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -21,6 +21,7 @@ # 3. This notice may not be removed or altered from any source distribution. import contextlib +import os import sqlite3 as sqlite import subprocess import sys @@ -749,7 +750,6 @@ def test_blob_seek_and_tell(self): self.blob.seek(10) self.assertEqual(self.blob.tell(), 10) - import os self.blob.seek(10, os.SEEK_SET) self.assertEqual(self.blob.tell(), 10) @@ -760,14 +760,16 @@ def test_blob_seek_and_tell(self): self.assertEqual(self.blob.tell(), 90) def test_blob_seek_error(self): - ops = ( - lambda: self.blob.seek(1000), - lambda: self.blob.seek(-10), - lambda: self.blob.seek(10, -1), + dataset = ( + (ValueError, lambda: self.blob.seek(1000)), + (ValueError, lambda: self.blob.seek(-10)), + (ValueError, lambda: self.blob.seek(10, -1)), + (OverflowError, lambda: self.blob.seek(2**65, os.SEEK_CUR)), + (OverflowError, lambda: self.blob.seek(2**65, os.SEEK_END)), ) - for op in ops: - with self.subTest(op=op): - self.assertRaises(ValueError, op) + for exc, fn in dataset: + with self.subTest(exc=exc, fn=fn): + self.assertRaises(exc, fn) def test_blob_read(self): buf = self.blob.read() @@ -907,6 +909,10 @@ def test_blob_set_slice_error(self): self.blob[5:10] = b"a" * 1000 with self.assertRaises(TypeError): del self.blob[5:10] + with self.assertRaises(BufferError): + self.blob[5:10] = memoryview(b"abcde")[::2] + with self.assertRaises(ValueError): + self.blob[5:10:0] = b"12345" def test_blob_sequence_not_supported(self): ops = ( @@ -950,6 +956,16 @@ def assign(): blob[0] = b"" with self.assertRaisesRegex(sqlite.ProgrammingError, msg): op() + def test_blob_close_bad_connection(self): + cx = sqlite.connect(":memory:") + cx.execute("create table test(b blob)") + cx.execute("insert into test values(zeroblob(1))") + blob = cx.open_blob("test", "b", 1) + cx.close() + self.assertRaisesRegex(sqlite.ProgrammingError, + "Cannot operate on a closed database", + blob.close) + def test_closed_blob_read(self): con = sqlite.connect(":memory:") con.execute("create table test(blob_col blob)") @@ -964,7 +980,8 @@ class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") self.cur = self.con.cursor() - self.cur.execute("create table test(name text)") + self.cur.execute("create table test(name text, b blob)") + self.cur.execute("insert into test values('blob', zeroblob(1))") def tearDown(self): self.cur.close() @@ -997,6 +1014,7 @@ def test_check_connection_thread(self): lambda: self.con.set_trace_callback(None), lambda: self.con.set_authorizer(None), lambda: self.con.create_collation("foo", None), + lambda: self.con.open_blob("test", "b", 1) ] for fn in fns: with self.subTest(fn=fn): From d9b5cdf9c979ad487a59f6cd408b522efc928642 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 17 Nov 2021 22:00:37 +0100 Subject: [PATCH 60/77] Format docs with linewidth 80 --- Doc/library/sqlite3.rst | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 4424f3309f78b1..54490b1c9de4d8 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -364,16 +364,15 @@ Connection Objects .. method:: open_blob(table, column, row, /, *, readonly=False, name="main") - On success a :class:`Blob` handle to the - :abbr:`BLOB (Binary Large OBject)` located in row *row*, - column *column*, table *table* in database *name* will be returned. - When *readonly* is :const:`True` the BLOB is opened without write - permissions. + On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large + OBject)` located in row *row*, column *column*, table *table* in database + *name* will be returned. When *readonly* is :const:`True` the BLOB is + opened without write permissions. .. note:: - The BLOB size cannot be changed using the :class:`Blob` class. Use - the SQL function ``zeroblob`` to create a blob with a fixed size. + The BLOB size cannot be changed using the :class:`Blob` class. Use the + SQL function ``zeroblob`` to create a blob with a fixed size. .. versionadded:: 3.11 @@ -1012,19 +1011,19 @@ Blob Objects .. class:: Blob - A :class:`Blob` instance can read and write the data in the - :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` class implements - the file and mapping protocols. For example, you can read data from the - :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. - You can call ``len(obj)`` to get size of the BLOB. + A :class:`Blob` instance can read and write the data in the :abbr:`BLOB + (Binary Large OBject)`. The :class:`Blob` class implements the file and + mapping protocols. For example, you can read data from the :class:`Blob` by + doing ``obj.read(5)`` or by doing ``obj[:5]``. Call ``len(obj)`` to get size + of the BLOB. .. method:: Blob.close() Close the BLOB. - The BLOB will be unusable from this point forward. An - :class:`~sqlite3.Error` (or subclass) exception will be - raised if any operation is attempted with the BLOB. + The BLOB will be unusable from this point forward. An + :class:`~sqlite3.Error` (or subclass) exception will be raised if any + operation is attempted with the BLOB. .. method:: Blob.__len__() @@ -1034,13 +1033,13 @@ Blob Objects Read *size* bytes of data from the BLOB at the current offset position. If the end of the BLOB is reached we will return the data up to end of - file. When *size* is not specified or is negative, :meth:`~Blob.read` + file. When *size* is not specified or is negative, :meth:`~Blob.read` will read till the end of the BLOB. .. method:: Blob.write(data, /) - Write *data* to the BLOB at the current offset. This function cannot - change the BLOB length. Writing beyond the end of the blob will result in + Write *data* to the BLOB at the current offset. This function cannot + change the BLOB length. Writing beyond the end of the blob will result in an exception being raised. .. method:: Blob.tell() @@ -1049,7 +1048,7 @@ Blob Objects .. method:: Blob.seek(offset, /, origin=sqlite3.BLOB_SEEK_START) - Set the current access position of the BLOB. The *origin* argument is + Set the current access position of the BLOB. The *origin* argument is optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values are :data:`os.SEEK_CUR` or 1 (seek relative to the current position) and :data:`os.SEEK_END` or 2 (seek relative to the @@ -1059,7 +1058,7 @@ Blob Objects .. literalinclude:: ../includes/sqlite3/blob.py - A :class:`Blob` can also be used with :term:`context manager`: + A :class:`Blob` can also be used as a :term:`context manager`: .. literalinclude:: ../includes/sqlite3/blob_with.py From aaa2721add9cb5ebb0927c52685189b3e7862741 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 17 Nov 2021 22:14:28 +0100 Subject: [PATCH 61/77] Improve tests --- Lib/test/test_sqlite3/test_dbapi.py | 42 +++++++++++++++++------------ Modules/_sqlite/blob.c | 2 +- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 992c196a74bce2..1aa7acb50a1cb1 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1093,9 +1093,10 @@ def test_blob_open_error(self): (("test", "notexisting", 1), {}), (("test", "blob_col", 2), {}), ) + regex = "no such" for args, kwds in dataset: with self.subTest(args=args, kwds=kwds): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaisesRegex(sqlite.OperationalError, regex): self.cx.open_blob(*args, **kwds) def test_blob_get_item(self): @@ -1106,14 +1107,14 @@ def test_blob_get_item_negative_index(self): def test_blob_get_item_error(self): dataset = ( - (b"", TypeError), - (105, IndexError), - (-105, IndexError), - (len(self.blob), IndexError), + (b"", TypeError, "Blob indices must be integers"), + (105, IndexError, "Blob index out of range"), + (-105, IndexError, "Blob index out of range"), + (len(self.blob), IndexError, "Blob index out of range"), ) - for idx, exc in dataset: - with self.subTest(idx=idx, exc=exc): - with self.assertRaises(exc): + for idx, exc, regex in dataset: + with self.subTest(idx=idx, exc=exc, regex=regex): + with self.assertRaisesRegex(exc, regex): self.blob[idx] def test_blob_get_slice(self): @@ -1123,7 +1124,7 @@ def test_blob_get_slice_negative_index(self): self.assertEqual(self.blob[5:-5], self.blob_data[5:-5]) def test_blob_get_slice_invalid_index(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "indices must be integers"): self.blob[5:b"a"] def test_blob_get_slice_with_skip(self): @@ -1132,32 +1133,38 @@ def test_blob_get_slice_with_skip(self): def test_blob_set_item(self): self.blob[0] = b"b" - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"b" + self.blob_data[1:]) + actual = self.cx.execute("select blob_col from test").fetchone()[0] + expected = b"b" + self.blob_data[1:] + self.assertEqual(actual, expected) def test_blob_set_item(self): self.blob[-1] = b"z" self.assertEqual(self.blob[-1], b"z") def test_blob_set_item_error(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "indices must be integers"): self.blob["a"] = b"b" - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, "must be a single byte"): self.blob[0] = b"abc" - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "doesn't support.*deletion"): del self.blob[0] - with self.assertRaises(IndexError): + with self.assertRaisesRegex(IndexError, "Blob index out of range"): self.blob[1000] = b"a" def test_blob_set_slice(self): self.blob[0:5] = b"bbbbb" - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bbbbb" + self.blob_data[5:]) + actual = self.cx.execute("select blob_col from test").fetchone()[0] + expected = b"bbbbb" + self.blob_data[5:] + self.assertEqual(actual, expected) def test_blob_set_empty_slice(self): self.blob[0:0] = b"" def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"bbbbb" - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bababababa" + self.blob_data[10:]) + actual = self.cx.execute("select blob_col from test").fetchone()[0] + expected = b"bababababa" + self.blob_data[10:] + self.assertEqual(actual, expected) def test_blob_get_empty_slice(self): self.assertEqual(self.blob[5:5], b"") @@ -1188,7 +1195,8 @@ def test_blob_context_manager(self): data = b"a" * 100 with self.cx.open_blob("test", "blob_col", 1) as blob: blob.write(data) - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) + actual = self.cx.execute("select blob_col from test").fetchone()[0] + self.assertEqual(actual, data) def test_blob_closed(self): cx = sqlite.connect(":memory:") diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 8dd3678c9f0ef7..57b66c82022eee 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -523,7 +523,7 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) return ass_subscript_slice(self, item, value); } - PyErr_SetString(PyExc_TypeError, "Blob indices must be integer"); + PyErr_SetString(PyExc_TypeError, "Blob indices must be integers"); return -1; } From 8f685ba2c1f42c853aa67e2e2bd622262563d30f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 2 Jan 2022 23:38:56 +0100 Subject: [PATCH 62/77] Add clinic_state() stub. (Currently not used) --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 57b66c82022eee..713aaec93f4b59 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -1,7 +1,7 @@ #include "blob.h" #include "util.h" -#define clinic_state() (pysqlite_get_state(NULL)) +#define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/blob.c.h" #undef clinic_state From 96df661180458fc4ad218c926903bb23e967424a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 2 Jan 2022 23:39:20 +0100 Subject: [PATCH 63/77] Visit/clear blob type during module traverse/clear --- Modules/_sqlite/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index c2444b9fdfbf6e..0eb64bfb12d64a 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -581,6 +581,7 @@ module_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->Warning); // Types + Py_VISIT(state->BlobType); Py_VISIT(state->ConnectionType); Py_VISIT(state->CursorType); Py_VISIT(state->PrepareProtocolType); @@ -613,6 +614,7 @@ module_clear(PyObject *module) Py_CLEAR(state->Warning); // Types + Py_CLEAR(state->BlobType); Py_CLEAR(state->ConnectionType); Py_CLEAR(state->CursorType); Py_CLEAR(state->PrepareProtocolType); From 97d12a8137c4dd02e96af571e7e25f76ee428fa8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 00:06:50 +0100 Subject: [PATCH 64/77] Harden some tests, simplify others --- Lib/test/test_sqlite3/test_dbapi.py | 176 +++++++++++++++------------- 1 file changed, 92 insertions(+), 84 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 1aa7acb50a1cb1..055bc8d4c13d18 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -993,18 +993,17 @@ def test_same_query_in_multiple_cursors(self): class BlobTests(unittest.TestCase): def setUp(self): self.cx = sqlite.connect(":memory:") - self.cx.execute("create table test(blob_col blob)") - self.blob_data = b"a" * 100 - self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data,)) - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.second_data = b"b" * 100 + self.cx.execute("create table test(b blob)") + self.data = b"this blob data string is exactly fifty bytes long!" + self.cx.execute("insert into test(b) values (?)", (self.data,)) + self.blob = self.cx.open_blob("test", "b", 1) def tearDown(self): self.blob.close() self.cx.close() def test_blob_length(self): - self.assertEqual(len(self.blob), 100) + self.assertEqual(len(self.blob), 50) def test_blob_seek_and_tell(self): self.blob.seek(10) @@ -1017,7 +1016,7 @@ def test_blob_seek_and_tell(self): self.assertEqual(self.blob.tell(), 20) self.blob.seek(-10, os.SEEK_END) - self.assertEqual(self.blob.tell(), 90) + self.assertEqual(self.blob.tell(), 40) def test_blob_seek_error(self): dataset = ( @@ -1033,65 +1032,71 @@ def test_blob_seek_error(self): def test_blob_read(self): buf = self.blob.read() - self.assertEqual(buf, self.blob_data) - self.assertEqual(len(buf), len(self.blob_data)) + self.assertEqual(buf, self.data) + self.assertEqual(len(buf), len(self.data)) def test_blob_read_too_much(self): - buf = self.blob.read(len(self.blob_data) * 2) - self.assertEqual(buf, self.blob_data) - self.assertEqual(len(buf), len(self.blob_data)) + buf = self.blob.read(len(self.data) * 2) + self.assertEqual(buf, self.data) + self.assertEqual(len(buf), len(self.data)) def test_blob_read_advance_offset(self): self.blob.read(10) self.assertEqual(self.blob.tell(), 10) def test_blob_read_start_at_offset(self): + new_data = b"b" * 50 self.blob.seek(10) - self.blob.write(self.second_data[:10]) + self.blob.write(new_data[:10]) self.blob.seek(10) - self.assertEqual(self.blob.read(10), self.second_data[:10]) + self.assertEqual(self.blob.read(10), new_data[:10]) def test_blob_write(self): - self.blob.write(self.second_data) - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], self.second_data) + new_data = b"new data".ljust(50) + self.blob.write(new_data) + row = self.cx.execute("select b from test").fetchone() + self.assertEqual(row[0], new_data) def test_blob_write_at_offset(self): - self.blob.seek(50) - self.blob.write(self.second_data[:50]) - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], - self.blob_data[:50] + self.second_data[:50]) + new_data = b"c" * 50 + self.blob.seek(25) + self.blob.write(new_data[:25]) + row = self.cx.execute("select b from test").fetchone() + self.assertEqual(row[0], self.data[:25] + new_data[:25]) def test_blob_write_advance_offset(self): - self.blob.write(self.second_data[:50]) - self.assertEqual(self.blob.tell(), 50) + new_data = b"d" * 50 + self.blob.write(new_data[:25]) + self.assertEqual(self.blob.tell(), 25) def test_blob_write_more_then_blob_size(self): - with self.assertRaises(ValueError): + msg = "data longer than blob length" + with self.assertRaisesRegex(ValueError, msg): self.blob.write(b"a" * 1000) def test_blob_read_after_row_change(self): - self.cx.execute("UPDATE test SET blob_col='aaaa' where rowid=1") + self.cx.execute("UPDATE test SET b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.read() def test_blob_write_after_row_change(self): - self.cx.execute("UPDATE test SET blob_col='aaaa' where rowid=1") + self.cx.execute("UPDATE test SET b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") def test_blob_write_when_readonly(self): read_only_blob = \ - self.cx.open_blob("test", "blob_col", 1, readonly=True) - with self.assertRaises(sqlite.OperationalError): + self.cx.open_blob("test", "b", 1, readonly=True) + with self.assertRaisesRegex(sqlite.OperationalError, "readonly"): read_only_blob.write(b"aaa") read_only_blob.close() def test_blob_open_error(self): dataset = ( - (("test", "blob_col", 1), {"name": "notexisting"}), - (("notexisting", "blob_col", 1), {}), + (("test", "b", 1), {"name": "notexisting"}), + (("notexisting", "b", 1), {}), (("test", "notexisting", 1), {}), - (("test", "blob_col", 2), {}), + (("test", "b", 2), {}), ) regex = "no such" for args, kwds in dataset: @@ -1100,10 +1105,11 @@ def test_blob_open_error(self): self.cx.open_blob(*args, **kwds) def test_blob_get_item(self): - self.assertEqual(self.blob[5], b"a") - - def test_blob_get_item_negative_index(self): - self.assertEqual(self.blob[-5], b"a") + self.assertEqual(self.blob[5], b"b") + self.assertEqual(self.blob[6], b"l") + self.assertEqual(self.blob[7], b"o") + self.assertEqual(self.blob[8], b"b") + self.assertEqual(self.blob[-1], b"!") def test_blob_get_item_error(self): dataset = ( @@ -1118,10 +1124,10 @@ def test_blob_get_item_error(self): self.blob[idx] def test_blob_get_slice(self): - self.assertEqual(self.blob[5:10], b"aaaaa") + self.assertEqual(self.blob[5:14], b"blob data") def test_blob_get_slice_negative_index(self): - self.assertEqual(self.blob[5:-5], self.blob_data[5:-5]) + self.assertEqual(self.blob[5:-5], self.data[5:-5]) def test_blob_get_slice_invalid_index(self): with self.assertRaisesRegex(TypeError, "indices must be integers"): @@ -1133,8 +1139,8 @@ def test_blob_get_slice_with_skip(self): def test_blob_set_item(self): self.blob[0] = b"b" - actual = self.cx.execute("select blob_col from test").fetchone()[0] - expected = b"b" + self.blob_data[1:] + actual = self.cx.execute("select b from test").fetchone()[0] + expected = b"b" + self.data[1:] self.assertEqual(actual, expected) def test_blob_set_item(self): @@ -1153,17 +1159,18 @@ def test_blob_set_item_error(self): def test_blob_set_slice(self): self.blob[0:5] = b"bbbbb" - actual = self.cx.execute("select blob_col from test").fetchone()[0] - expected = b"bbbbb" + self.blob_data[5:] + actual = self.cx.execute("select b from test").fetchone()[0] + expected = b"bbbbb" + self.data[5:] self.assertEqual(actual, expected) def test_blob_set_empty_slice(self): self.blob[0:0] = b"" + self.assertEqual(self.blob[:], self.data) def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"bbbbb" - actual = self.cx.execute("select blob_col from test").fetchone()[0] - expected = b"bababababa" + self.blob_data[10:] + actual = self.cx.execute("select b from test").fetchone()[0] + expected = b"bhbsbbbob " + self.data[10:] self.assertEqual(actual, expected) def test_blob_get_empty_slice(self): @@ -1192,56 +1199,57 @@ def test_blob_sequence_not_supported(self): self.assertRaises(TypeError, op) def test_blob_context_manager(self): - data = b"a" * 100 - with self.cx.open_blob("test", "blob_col", 1) as blob: + data = b"a" * 50 + with self.cx.open_blob("test", "b", 1) as blob: blob.write(data) - actual = self.cx.execute("select blob_col from test").fetchone()[0] + actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, data) def test_blob_closed(self): - cx = sqlite.connect(":memory:") - cx.execute("create table test(b blob)") - cx.execute("insert into test values (zeroblob(100))") - blob = cx.open_blob("test", "b", 1) - blob.close() - - def assign(): blob[0] = b"" - ops = [ - lambda: blob.read(), - lambda: blob.write(b""), - lambda: blob.seek(0), - lambda: blob.tell(), - lambda: blob.__enter__(), - lambda: blob.__exit__(None, None, None), - lambda: len(blob), - lambda: blob[0], - lambda: blob[0:1], - assign, - ] - msg = "Cannot operate on a closed blob" - for op in ops: - with self.subTest(op=op): - with self.assertRaisesRegex(sqlite.ProgrammingError, msg): - op() + with memory_database() as cx: + cx.execute("create table test(b blob)") + cx.execute("insert into test values (zeroblob(100))") + blob = cx.open_blob("test", "b", 1) + blob.close() + + def assign(): blob[0] = b"" + ops = [ + lambda: blob.read(), + lambda: blob.write(b""), + lambda: blob.seek(0), + lambda: blob.tell(), + lambda: blob.__enter__(), + lambda: blob.__exit__(None, None, None), + lambda: len(blob), + lambda: blob[0], + lambda: blob[0:1], + assign, + ] + msg = "Cannot operate on a closed blob" + for op in ops: + with self.subTest(op=op): + with self.assertRaisesRegex(sqlite.ProgrammingError, msg): + op() def test_blob_close_bad_connection(self): - cx = sqlite.connect(":memory:") - cx.execute("create table test(b blob)") - cx.execute("insert into test values(zeroblob(1))") - blob = cx.open_blob("test", "b", 1) - cx.close() - self.assertRaisesRegex(sqlite.ProgrammingError, - "Cannot operate on a closed database", - blob.close) + with memory_database() as cx: + cx.execute("create table test(b blob)") + cx.execute("insert into test values(zeroblob(1))") + blob = cx.open_blob("test", "b", 1) + cx.close() + self.assertRaisesRegex(sqlite.ProgrammingError, + "Cannot operate on a closed database", + blob.close) def test_closed_blob_read(self): - con = sqlite.connect(":memory:") - con.execute("create table test(blob_col blob)") - con.execute("insert into test(blob_col) values (zeroblob(100))") - blob = con.open_blob("test", "blob_col", 1) - con.close() - with self.assertRaises(sqlite.ProgrammingError): - blob.read() + with memory_database() as cx: + cx.execute("create table test(b blob)") + cx.execute("insert into test(b) values (zeroblob(100))") + blob = cx.open_blob("test", "b", 1) + cx.close() + self.assertRaisesRegex(sqlite.ProgrammingError, + "Cannot operate on a closed database", + blob.read) class ThreadTests(unittest.TestCase): From 2e63b3ef0002caf40c43996e69dea7b7a83b387a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 13:49:52 +0100 Subject: [PATCH 65/77] Simplify example --- Doc/includes/sqlite3/blob_with.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index 2e48d568830820..67eef36cb4f6e2 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -2,7 +2,7 @@ con = sqlite3.connect(":memory:") -con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") with con.open_blob("test", "blob_col", 1) as blob: From be2774789d6dfab1a18b0e1642fdef2df6cf6edf Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:27:22 +0100 Subject: [PATCH 66/77] Update docs - sync with docstrings - simplify some sections - clarify some parameters --- Doc/library/sqlite3.rst | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e60ffa08dafe34..32ec855cdcf791 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -382,9 +382,9 @@ Connection Objects .. method:: open_blob(table, column, row, /, *, readonly=False, name="main") - On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large + On success, a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, column *column*, table *table* in database - *name* will be returned. When *readonly* is :const:`True` the BLOB is + *name* will be returned. When *readonly* is :const:`True` the blob is opened without write permissions. .. note:: @@ -1029,11 +1029,9 @@ Blob Objects .. class:: Blob - A :class:`Blob` instance can read and write the data in the :abbr:`BLOB + A :class:`Blob` instance can read and write the data in a :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` class implements the file and - mapping protocols. For example, you can read data from the :class:`Blob` by - doing ``obj.read(5)`` or by doing ``obj[:5]``. Call ``len(obj)`` to get size - of the BLOB. + mapping protocols. .. method:: Blob.close() @@ -1041,15 +1039,15 @@ Blob Objects The BLOB will be unusable from this point forward. An :class:`~sqlite3.Error` (or subclass) exception will be raised if any - operation is attempted with the BLOB. + further operation is attempted with the BLOB. .. method:: Blob.__len__() - Return the BLOB size. + Return the BLOB size as length in bytes. - .. method:: Blob.read([size]) + .. method:: Blob.read(length=-1, /) - Read *size* bytes of data from the BLOB at the current offset position. + Read *length* bytes of data from the BLOB at the current offset position. If the end of the BLOB is reached we will return the data up to end of file. When *size* is not specified or is negative, :meth:`~Blob.read` will read till the end of the BLOB. @@ -1064,13 +1062,13 @@ Blob Objects Return the current access position of the BLOB. - .. method:: Blob.seek(offset, /, origin=sqlite3.BLOB_SEEK_START) + .. method:: Blob.seek(offset, origin=sqlite3.BLOB_SEEK_START, /) - Set the current access position of the BLOB. The *origin* argument is - optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB - positioning); other values are :data:`os.SEEK_CUR` or 1 (seek relative to - the current position) and :data:`os.SEEK_END` or 2 (seek relative to the - BLOB’s end). + Set the current access position of the BLOB to *offset*. The *origin* + argument defaults to :data:`os.SEEK_SET` (absolute BLOB positioning). + Other values for *origin* are :data:`os.SEEK_CUR` (seek relative to the + current position) and :data:`os.SEEK_END` (seek relative to the BLOB’s + end). :class:`Blob` example: From 5add365e2026aa79240180a911e75ac7a2b0194c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:37:26 +0100 Subject: [PATCH 67/77] Simplify tests --- Lib/test/test_sqlite3/test_dbapi.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 055bc8d4c13d18..824557b6d96efc 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1085,11 +1085,10 @@ def test_blob_write_after_row_change(self): self.blob.write(b"aaa") def test_blob_write_when_readonly(self): - read_only_blob = \ - self.cx.open_blob("test", "b", 1, readonly=True) + ro_blob = self.cx.open_blob("test", "b", 1, readonly=True) with self.assertRaisesRegex(sqlite.OperationalError, "readonly"): - read_only_blob.write(b"aaa") - read_only_blob.close() + ro_blob.write(b"aaa") + ro_blob.close() def test_blob_open_error(self): dataset = ( @@ -1134,13 +1133,12 @@ def test_blob_get_slice_invalid_index(self): self.blob[5:b"a"] def test_blob_get_slice_with_skip(self): - self.blob.write(b"abcdefghij") - self.assertEqual(self.blob[0:10:2], b"acegi") + self.assertEqual(self.blob[0:10:2], b"ti lb") def test_blob_set_item(self): self.blob[0] = b"b" - actual = self.cx.execute("select b from test").fetchone()[0] expected = b"b" + self.data[1:] + actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, expected) def test_blob_set_item(self): @@ -1159,8 +1157,8 @@ def test_blob_set_item_error(self): def test_blob_set_slice(self): self.blob[0:5] = b"bbbbb" - actual = self.cx.execute("select b from test").fetchone()[0] expected = b"bbbbb" + self.data[5:] + actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, expected) def test_blob_set_empty_slice(self): From bacf08726483b7fcddb0729f9b51a1041d393d90 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:41:26 +0100 Subject: [PATCH 68/77] Safe close --- Modules/_sqlite/blob.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 713aaec93f4b59..e2d2a4c9e3ab1c 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -16,10 +16,12 @@ static void close_blob(pysqlite_Blob *self) { if (self->blob) { + sqlite3_blob *blob = self->blob; + self->blob = NULL; + Py_BEGIN_ALLOW_THREADS - sqlite3_blob_close(self->blob); + sqlite3_blob_close(blob); Py_END_ALLOW_THREADS - self->blob = NULL; } } From 5ff202d3189c87ac7b8ebcd4010e378caa7afa28 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:50:39 +0100 Subject: [PATCH 69/77] Condense a comment --- Modules/_sqlite/blob.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index e2d2a4c9e3ab1c..0a070388e753dd 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,7 +11,6 @@ class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ - static void close_blob(pysqlite_Blob *self) { @@ -56,11 +55,7 @@ blob_dealloc(pysqlite_Blob *self) Py_DECREF(tp); } -/* - * Checks if a blob object is usable (i. e. not closed). - * - * 0 => error; 1 => ok - */ +// Return 1 if the blob object is usable, 0 if not. static int check_blob(pysqlite_Blob *self) { From 1fa5901dbeddf003a07fdd2a069742d260d58605 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:56:10 +0100 Subject: [PATCH 70/77] Make sure we catch int overflow for all types of writes --- Modules/_sqlite/blob.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 0a070388e753dd..e93e606e962290 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -187,8 +187,12 @@ blob_read_impl(pysqlite_Blob *self, int length) static int inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) { - int rc; + if (len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); + return -1; + } + int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_write(self->blob, buf, len, offset); Py_END_ALLOW_THREADS @@ -218,11 +222,6 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) return NULL; } - if (data->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); - return NULL; - } - int remaining_len = sqlite3_blob_bytes(self->blob) - self->offset; if (data->len > remaining_len) { PyErr_SetString(PyExc_ValueError, "data longer than blob length"); From 7aec288c269d70af04ea28fdda305e6f4cfb7cf9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 18 Jan 2022 23:22:49 +0100 Subject: [PATCH 71/77] Fix typo in test name --- Lib/test/test_sqlite3/test_dbapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 824557b6d96efc..40fd7147407a8a 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1069,7 +1069,7 @@ def test_blob_write_advance_offset(self): self.blob.write(new_data[:25]) self.assertEqual(self.blob.tell(), 25) - def test_blob_write_more_then_blob_size(self): + def test_blob_write_more_than_blob_size(self): msg = "data longer than blob length" with self.assertRaisesRegex(ValueError, msg): self.blob.write(b"a" * 1000) From aaaf7abca33d916be6b8ebe8e171a9b96bf047a2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 18 Jan 2022 23:31:21 +0100 Subject: [PATCH 72/77] Improve test --- Lib/test/test_sqlite3/test_dbapi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 40fd7147407a8a..30c5ee44776c4d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1041,8 +1041,10 @@ def test_blob_read_too_much(self): self.assertEqual(len(buf), len(self.data)) def test_blob_read_advance_offset(self): - self.blob.read(10) - self.assertEqual(self.blob.tell(), 10) + n = 10 + buf = self.blob.read(n) + self.assertEqual(buf, self.data[:n]) + self.assertEqual(self.blob.tell(), n) def test_blob_read_start_at_offset(self): new_data = b"b" * 50 From f9e65c011d252b5f360ecad1fdb256a517654b94 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 00:30:56 +0100 Subject: [PATCH 73/77] Improve tests --- Lib/test/test_sqlite3/test_dbapi.py | 49 ++++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 30c5ee44776c4d..9217d278c2c276 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -20,6 +20,7 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +import _testcapi import contextlib import os import sqlite3 as sqlite @@ -1019,16 +1020,26 @@ def test_blob_seek_and_tell(self): self.assertEqual(self.blob.tell(), 40) def test_blob_seek_error(self): + msg_oor = "offset out of blob range" + msg_orig = "'origin' should be 0, 1, or 2" + msg_of = "seek offset result in overflow" + dataset = ( - (ValueError, lambda: self.blob.seek(1000)), - (ValueError, lambda: self.blob.seek(-10)), - (ValueError, lambda: self.blob.seek(10, -1)), - (OverflowError, lambda: self.blob.seek(2**65, os.SEEK_CUR)), - (OverflowError, lambda: self.blob.seek(2**65, os.SEEK_END)), + (ValueError, msg_oor, lambda: self.blob.seek(1000)), + (ValueError, msg_oor, lambda: self.blob.seek(-10)), + (ValueError, msg_orig, lambda: self.blob.seek(10, -1)), + (ValueError, msg_orig, lambda: self.blob.seek(10, 3)), ) - for exc, fn in dataset: - with self.subTest(exc=exc, fn=fn): - self.assertRaises(exc, fn) + for exc, msg, fn in dataset: + with self.subTest(exc=exc, msg=msg, fn=fn): + self.assertRaisesRegex(exc, msg, fn) + + n = len(self.data) // 2 + self.blob.seek(n, os.SEEK_SET) + with self.assertRaisesRegex(OverflowError, msg_of): + self.blob.seek(_testcapi.INT_MAX, os.SEEK_CUR) + with self.assertRaisesRegex(OverflowError, msg_of): + self.blob.seek( _testcapi.INT_MAX, os.SEEK_END) def test_blob_read(self): buf = self.blob.read() @@ -1053,6 +1064,11 @@ def test_blob_read_start_at_offset(self): self.blob.seek(10) self.assertEqual(self.blob.read(10), new_data[:10]) + def test_blob_read_after_row_change(self): + self.cx.execute("update test set b='aaaa' where rowid=1") + with self.assertRaises(sqlite.OperationalError): + self.blob.read() + def test_blob_write(self): new_data = b"new data".ljust(50) self.blob.write(new_data) @@ -1071,22 +1087,16 @@ def test_blob_write_advance_offset(self): self.blob.write(new_data[:25]) self.assertEqual(self.blob.tell(), 25) - def test_blob_write_more_than_blob_size(self): - msg = "data longer than blob length" - with self.assertRaisesRegex(ValueError, msg): + def test_blob_write_error_length(self): + with self.assertRaisesRegex(ValueError, "data longer than blob"): self.blob.write(b"a" * 1000) - def test_blob_read_after_row_change(self): - self.cx.execute("UPDATE test SET b='aaaa' where rowid=1") - with self.assertRaises(sqlite.OperationalError): - self.blob.read() - - def test_blob_write_after_row_change(self): - self.cx.execute("UPDATE test SET b='aaaa' where rowid=1") + def test_blob_write_error_row_changed(self): + self.cx.execute("update test set b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") - def test_blob_write_when_readonly(self): + def test_blob_write_error_readonly(self): ro_blob = self.cx.open_blob("test", "b", 1, readonly=True) with self.assertRaisesRegex(sqlite.OperationalError, "readonly"): ro_blob.write(b"aaa") @@ -1117,6 +1127,7 @@ def test_blob_get_item_error(self): (b"", TypeError, "Blob indices must be integers"), (105, IndexError, "Blob index out of range"), (-105, IndexError, "Blob index out of range"), + (_testcapi.ULONG_MAX, IndexError, "cannot fit 'int'"), (len(self.blob), IndexError, "Blob index out of range"), ) for idx, exc, regex in dataset: From 6a5c86432aa74c460bbc146d34bbb51e0523ba37 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 00:31:13 +0100 Subject: [PATCH 74/77] Always check length in inner write --- Modules/_sqlite/blob.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index e93e606e962290..34f13e1ad77d43 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -187,8 +187,9 @@ blob_read_impl(pysqlite_Blob *self, int length) static int inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) { - if (len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); + int remaining_len = sqlite3_blob_bytes(self->blob) - self->offset; + if (len > remaining_len) { + PyErr_SetString(PyExc_ValueError, "data longer than blob length"); return -1; } @@ -222,12 +223,6 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) return NULL; } - int remaining_len = sqlite3_blob_bytes(self->blob) - self->offset; - if (data->len > remaining_len) { - PyErr_SetString(PyExc_ValueError, "data longer than blob length"); - return NULL; - } - int rc = inner_write(self, data->buf, data->len, self->offset); if (rc < 0) { return NULL; From ea1045ca7ec82ae8fa7784a806ae0416a557d75a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 00:42:03 +0100 Subject: [PATCH 75/77] Fix missing array sentinel --- Modules/_sqlite/blob.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 34f13e1ad77d43..5fa4cd3f98b63d 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -532,6 +532,7 @@ static PyMethodDef blob_methods[] = { static struct PyMemberDef blob_members[] = { {"__weaklistoffset__", T_PYSSIZET, offsetof(pysqlite_Blob, in_weakreflist), READONLY}, + {NULL}, }; static PyType_Slot blob_slots[] = { From 2f848d936e6bde7bfe8e795bd18e022b605c9700 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 01:10:56 +0100 Subject: [PATCH 76/77] Match apsw's API open_blob => blobopen --- Doc/includes/sqlite3/blob.py | 2 +- Doc/includes/sqlite3/blob_with.py | 2 +- Doc/library/sqlite3.rst | 2 +- Doc/whatsnew/3.11.rst | 2 +- Lib/test/test_sqlite3/test_dbapi.py | 16 +++++------ .../2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 2 +- Modules/_sqlite/clinic/connection.c.h | 27 +++++++++---------- Modules/_sqlite/connection.c | 11 ++++---- 8 files changed, 31 insertions(+), 33 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index f928349fd0e981..61994fb82dd72a 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -4,7 +4,7 @@ con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") -blob = con.open_blob("test", "blob_col", 1) +blob = con.blobopen("test", "blob_col", 1) blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index 67eef36cb4f6e2..d489bd632d867d 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -5,7 +5,7 @@ con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") -with con.open_blob("test", "blob_col", 1) as blob: +with con.blobopen("test", "blob_col", 1) as blob: blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index a332d4841b4726..8c860e15a618b4 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -394,7 +394,7 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. - .. method:: open_blob(table, column, row, /, *, readonly=False, name="main") + .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") On success, a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, column *column*, table *table* in database diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index c193d4542c7996..01f03bfbdf4011 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -301,7 +301,7 @@ sqlite3 Instead we leave it to the SQLite library to handle these cases. (Contributed by Erlend E. Aasland in :issue:`44092`.) -* Add :meth:`~sqlite3.Connection.open_blob` to :class:`sqlite3.Connection`. +* Add :meth:`~sqlite3.Connection.blobopen` to :class:`sqlite3.Connection`. :class:`sqlite3.Blob` allows incremental I/O operations on blobs. (Contributed by Aviv Palivoda and Erlend E. Aasland in :issue:`24905`) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 9217d278c2c276..9bf2e54e6dcb06 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -997,7 +997,7 @@ def setUp(self): self.cx.execute("create table test(b blob)") self.data = b"this blob data string is exactly fifty bytes long!" self.cx.execute("insert into test(b) values (?)", (self.data,)) - self.blob = self.cx.open_blob("test", "b", 1) + self.blob = self.cx.blobopen("test", "b", 1) def tearDown(self): self.blob.close() @@ -1097,7 +1097,7 @@ def test_blob_write_error_row_changed(self): self.blob.write(b"aaa") def test_blob_write_error_readonly(self): - ro_blob = self.cx.open_blob("test", "b", 1, readonly=True) + ro_blob = self.cx.blobopen("test", "b", 1, readonly=True) with self.assertRaisesRegex(sqlite.OperationalError, "readonly"): ro_blob.write(b"aaa") ro_blob.close() @@ -1113,7 +1113,7 @@ def test_blob_open_error(self): for args, kwds in dataset: with self.subTest(args=args, kwds=kwds): with self.assertRaisesRegex(sqlite.OperationalError, regex): - self.cx.open_blob(*args, **kwds) + self.cx.blobopen(*args, **kwds) def test_blob_get_item(self): self.assertEqual(self.blob[5], b"b") @@ -1211,7 +1211,7 @@ def test_blob_sequence_not_supported(self): def test_blob_context_manager(self): data = b"a" * 50 - with self.cx.open_blob("test", "b", 1) as blob: + with self.cx.blobopen("test", "b", 1) as blob: blob.write(data) actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, data) @@ -1220,7 +1220,7 @@ def test_blob_closed(self): with memory_database() as cx: cx.execute("create table test(b blob)") cx.execute("insert into test values (zeroblob(100))") - blob = cx.open_blob("test", "b", 1) + blob = cx.blobopen("test", "b", 1) blob.close() def assign(): blob[0] = b"" @@ -1246,7 +1246,7 @@ def test_blob_close_bad_connection(self): with memory_database() as cx: cx.execute("create table test(b blob)") cx.execute("insert into test values(zeroblob(1))") - blob = cx.open_blob("test", "b", 1) + blob = cx.blobopen("test", "b", 1) cx.close() self.assertRaisesRegex(sqlite.ProgrammingError, "Cannot operate on a closed database", @@ -1256,7 +1256,7 @@ def test_closed_blob_read(self): with memory_database() as cx: cx.execute("create table test(b blob)") cx.execute("insert into test(b) values (zeroblob(100))") - blob = cx.open_blob("test", "b", 1) + blob = cx.blobopen("test", "b", 1) cx.close() self.assertRaisesRegex(sqlite.ProgrammingError, "Cannot operate on a closed database", @@ -1303,7 +1303,7 @@ def test_check_connection_thread(self): lambda: self.con.create_collation("foo", None), lambda: self.con.setlimit(sqlite.SQLITE_LIMIT_LENGTH, -1), lambda: self.con.getlimit(sqlite.SQLITE_LIMIT_LENGTH), - lambda: self.con.open_blob("test", "b", 1), + lambda: self.con.blobopen("test", "b", 1), ] for fn in fns: with self.subTest(fn=fn): diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst index c9571f374862f0..0a57f90c12378f 100644 --- a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -1,3 +1,3 @@ -Add :meth:`~sqlite3.Connection.open_blob` to :class:`sqlite3.Connection`. +Add :meth:`~sqlite3.Connection.blobopen` to :class:`sqlite3.Connection`. :class:`sqlite3.Blob` allows incremental I/O operations on blobs. Patch by Aviv Palivoda and Erlend E. Aasland. diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index acadbe3b7eae49..c2a45594ca907b 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -145,26 +145,25 @@ pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ return return_value; } -PyDoc_STRVAR(pysqlite_connection_open_blob__doc__, -"open_blob($self, table, column, row, /, *, readonly=False, name=\'main\')\n" +PyDoc_STRVAR(blobopen__doc__, +"blobopen($self, table, column, row, /, *, readonly=False, name=\'main\')\n" "--\n" "\n" "Return a blob object. Non-standard."); -#define PYSQLITE_CONNECTION_OPEN_BLOB_METHODDEF \ - {"open_blob", (PyCFunction)(void(*)(void))pysqlite_connection_open_blob, METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_open_blob__doc__}, +#define BLOBOPEN_METHODDEF \ + {"blobopen", (PyCFunction)(void(*)(void))blobopen, METH_FASTCALL|METH_KEYWORDS, blobopen__doc__}, static PyObject * -pysqlite_connection_open_blob_impl(pysqlite_Connection *self, - const char *table, const char *col, - int row, int readonly, const char *name); +blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, + int row, int readonly, const char *name); static PyObject * -pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"", "", "", "readonly", "name", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "open_blob", 0}; + static _PyArg_Parser _parser = {NULL, _keywords, "blobopen", 0}; PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; const char *table; @@ -178,7 +177,7 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, goto exit; } if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("open_blob", "argument 1", "str", args[0]); + _PyArg_BadArgument("blobopen", "argument 1", "str", args[0]); goto exit; } Py_ssize_t table_length; @@ -191,7 +190,7 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, goto exit; } if (!PyUnicode_Check(args[1])) { - _PyArg_BadArgument("open_blob", "argument 2", "str", args[1]); + _PyArg_BadArgument("blobopen", "argument 2", "str", args[1]); goto exit; } Py_ssize_t col_length; @@ -220,7 +219,7 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, } } if (!PyUnicode_Check(args[4])) { - _PyArg_BadArgument("open_blob", "argument 'name'", "str", args[4]); + _PyArg_BadArgument("blobopen", "argument 'name'", "str", args[4]); goto exit; } Py_ssize_t name_length; @@ -233,7 +232,7 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, goto exit; } skip_optional_kwonly: - return_value = pysqlite_connection_open_blob_impl(self, table, col, row, readonly, name); + return_value = blobopen_impl(self, table, col, row, readonly, name); exit: return return_value; @@ -930,4 +929,4 @@ getlimit(pysqlite_Connection *self, PyObject *arg) #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */ -/*[clinic end generated code: output=ecc1c000c7217a90 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=959de9d109b85d3f input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 3d7b9227ebc256..bf6b5c78d0b1b8 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -438,7 +438,7 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) } /*[clinic input] -_sqlite3.Connection.open_blob as pysqlite_connection_open_blob +_sqlite3.Connection.blobopen as blobopen table: str column as col: str @@ -452,10 +452,9 @@ Return a blob object. Non-standard. [clinic start generated code]*/ static PyObject * -pysqlite_connection_open_blob_impl(pysqlite_Connection *self, - const char *table, const char *col, - int row, int readonly, const char *name) -/*[clinic end generated code: output=5c11d18e7eb629ef input=9192dd28146d4e14]*/ +blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, + int row, int readonly, const char *name) +/*[clinic end generated code: output=0c8e2e58516d0b5c input=1eec15d2b87bf09e]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2041,13 +2040,13 @@ static PyMethodDef connection_methods[] = { PYSQLITE_CONNECTION_INTERRUPT_METHODDEF PYSQLITE_CONNECTION_ITERDUMP_METHODDEF PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF - PYSQLITE_CONNECTION_OPEN_BLOB_METHODDEF PYSQLITE_CONNECTION_ROLLBACK_METHODDEF PYSQLITE_CONNECTION_SET_AUTHORIZER_METHODDEF PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF PYSQLITE_CONNECTION_SET_TRACE_CALLBACK_METHODDEF SETLIMIT_METHODDEF GETLIMIT_METHODDEF + BLOBOPEN_METHODDEF {NULL, NULL} }; From 8d906bca4574d2adb0862b86e9e537d73498188c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 01:33:24 +0100 Subject: [PATCH 77/77] Fix overflow test on win x64 --- Lib/test/test_sqlite3/test_dbapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 9bf2e54e6dcb06..b25e6b125f9095 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1127,7 +1127,7 @@ def test_blob_get_item_error(self): (b"", TypeError, "Blob indices must be integers"), (105, IndexError, "Blob index out of range"), (-105, IndexError, "Blob index out of range"), - (_testcapi.ULONG_MAX, IndexError, "cannot fit 'int'"), + (_testcapi.ULLONG_MAX, IndexError, "cannot fit 'int'"), (len(self.blob), IndexError, "Blob index out of range"), ) for idx, exc, regex in dataset: pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy