From 4fa0e91bd77290614e721718691bb76ac2dc7f0e Mon Sep 17 00:00:00 2001 From: palaviv Date: Fri, 24 Feb 2017 12:23:19 +0200 Subject: [PATCH 01/18] support BLOB incremental I/O in sqlite module --- Doc/includes/sqlite3/blob.py | 13 ++ Doc/includes/sqlite3/blob_with.py | 12 ++ Doc/library/sqlite3.rst | 67 ++++++ Lib/sqlite3/test/dbapi.py | 279 +++++++++++++++++++++++- Modules/_sqlite/blob.c | 342 ++++++++++++++++++++++++++++++ Modules/_sqlite/blob.h | 25 +++ Modules/_sqlite/connection.c | 93 +++++++- Modules/_sqlite/connection.h | 3 +- Modules/_sqlite/module.c | 4 +- setup.py | 3 +- 10 files changed, 830 insertions(+), 11 deletions(-) create mode 100644 Doc/includes/sqlite3/blob.py create mode 100644 Doc/includes/sqlite3/blob_with.py create mode 100644 Modules/_sqlite/blob.c create mode 100644 Modules/_sqlite/blob.h diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py new file mode 100644 index 00000000000000..b8d2c78d1a365d --- /dev/null +++ b/Doc/includes/sqlite3/blob.py @@ -0,0 +1,13 @@ +import sqlite3 + +con = sqlite3.connect(":memory:") +# creating the table +con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("insert into test(blob_col) values (zeroblob(10))") +# opening blob handle +blob = con.open_blob("test", "blob_col", 1, 1) +blob.write(b"a" * 5) +blob.write(b"b" * 5) +blob.seek(0) +print(blob.read()) # will print b"aaaaabbbbb" +blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py new file mode 100644 index 00000000000000..624b680591901a --- /dev/null +++ b/Doc/includes/sqlite3/blob_with.py @@ -0,0 +1,12 @@ +import sqlite3 + +con = sqlite3.connect(":memory:") +# creating the table +con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("insert into test(blob_col) values (zeroblob(10))") +# opening blob handle +with con.open_blob("test", "blob_col", 1, 1) as blob: + blob.write(b"a" * 5) + blob.write(b"b" * 5) + blob.seek(0) + print(blob.read()) # will print b"aaaaabbbbb" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ccb82278bdaa13..ceb92b3fe90fab 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -301,6 +301,16 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. + .. method:: open_blob(table, column, row, readonly=False, dbname="main") + + On success a :class:`Blob` handle to the + :abbr:`BLOB (Binary Large OBject)` located in row *row*, + column *column*, table *table* in database *dbname* will be returned. + When *readonly* is :const:`True` the BLOB is opened with read + permissions. Otherwise the BLOB has read and write permissions. + + .. versionadded:: 3.7 + .. method:: commit() This method commits the current transaction. If you don't call this method, @@ -853,6 +863,63 @@ Exceptions transactions turned off. It is a subclass of :exc:`DatabaseError`. +.. _sqlite3-blob-objects: + +Blob Objects +------------ + +.. versionadded:: 3.7 + +.. class:: Blob + + A :class:`Blob` instance can read and write the data in the + :abbr:`BLOB (Binary Large OBject)`. + + .. method:: Blob.close() + + Close the BLOB now (rather than whenever __del__ is called). + + The BLOB will be unusable from this point forward; an + :class:`~sqlite3.Error` (or subclass) exception will be + raised if any operation is attempted with the BLOB. + + .. method:: Blob.__len__() + + Return the BLOB size. + + .. method:: Blob.read([size]) + + Read *size* bytes of data from the BLOB at the current offset position. + If the end of the BLOB is reached we will return the data up to end of + file. When *size* is not specified or negative we will read up to end + of BLOB. + + .. method:: Blob.write(data) + + Write *data* to the BLOB at the current offset. This function cannot + changed BLOB length. If data write will result in writing to more + then BLOB current size an error will be raised. + + .. method:: Blob.tell() + + Return the current offset of the BLOB. + + .. method:: Blob.seek(offset, [whence]) + + Set the BLOB offset. The *whence* argument is optional and defaults to + :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values + are :data:`os.SEEK_CUR` or 1 (seek relative to the current position) and + :data:`os.SEEK_END` or 2 (seek relative to the BLOB’s end). + + :class:`Blob` example: + + .. literalinclude:: ../includes/sqlite3/blob.py + + A :class:`Blob` can also be used with :term:`context manager`: + + .. literalinclude:: ../includes/sqlite3/blob_with.py + + .. _sqlite3-types: SQLite and Python types diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index be11337154bdd2..85db83ee51f246 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -511,6 +511,167 @@ def CheckLastRowIDInsertOR(self): self.assertEqual(results, expected) +class BlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.blob_data = b"a" * 100 + self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data, )) + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.second_data = b"b" * 100 + + def tearDown(self): + self.blob.close() + self.cx.close() + + def CheckLength(self): + self.assertEqual(len(self.blob), 100) + + def CheckTell(self): + self.assertEqual(self.blob.tell(), 0) + + def CheckSeekFromBlobStart(self): + self.blob.seek(10) + self.assertEqual(self.blob.tell(), 10) + self.blob.seek(10, 0) + self.assertEqual(self.blob.tell(), 10) + + def CheckSeekFromCurrentPosition(self): + self.blob.seek(10, 1) + self.blob.seek(10, 1) + self.assertEqual(self.blob.tell(), 20) + + def CheckSeekFromBlobEnd(self): + self.blob.seek(-10, 2) + self.assertEqual(self.blob.tell(), 90) + + def CheckBlobSeekOverBlobSize(self): + try: + self.blob.seek(1000) + self.fail("should have raised a ValueError") + except ValueError: + pass + except Exception: + self.fail("should have raised a ValueError") + + def CheckBlobSeekUnderBlobSize(self): + try: + self.blob.seek(-10) + self.fail("should have raised a ValueError") + except ValueError: + pass + except Exception: + self.fail("should have raised a ValueError") + + def CheckBlobRead(self): + self.assertEqual(self.blob.read(), self.blob_data) + + def CheckBlobReadSize(self): + self.assertEqual(len(self.blob.read(10)), 10) + + def CheckBlobReadAdvanceOffset(self): + self.blob.read(10) + self.assertEqual(self.blob.tell(), 10) + + def CheckBlobReadStartAtOffset(self): + self.blob.seek(10) + self.blob.write(self.second_data[:10]) + self.blob.seek(10) + self.assertEqual(self.blob.read(10), self.second_data[:10]) + + def CheckBlobWrite(self): + self.blob.write(self.second_data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], self.second_data) + + def CheckBlobWriteAtOffset(self): + self.blob.seek(50) + self.blob.write(self.second_data[:50]) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], + self.blob_data[:50] + self.second_data[:50]) + + def CheckBlobWriteAdvanceOffset(self): + self.blob.write(self.second_data[:50]) + self.assertEqual(self.blob.tell(), 50) + + def CheckBlobWriteMoreThenBlobSize(self): + try: + self.blob.write(b"a" * 1000) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobReadAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + try: + self.blob.read() + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobWriteAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + try: + self.blob.write(b"aaa") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobWriteWhenReadOnly(self): + read_only_blob = \ + self.cx.open_blob("test", "blob_col", 1, readonly=True) + try: + read_only_blob.write(b"aaa") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + read_only_blob.close() + + def CheckBlobOpenWithBadDb(self): + try: + self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadTable(self): + try: + self.cx.open_blob("notexisintg", "blob_col", 1) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadColumn(self): + try: + self.cx.open_blob("test", "notexisting", 1) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadRow(self): + try: + self.cx.open_blob("test", "blob_col", 2) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + +@unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -768,6 +929,20 @@ def CheckClosedCurExecute(self): with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4") + def CheckClosedBlobRead(self): + con = sqlite.connect(":memory:") + con.execute("create table test(id integer primary key, blob_col blob)") + con.execute("insert into test(blob_col) values (zeroblob(100))") + blob = con.open_blob("test", "blob_col", 1) + con.close() + try: + blob.read() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except: + self.fail("Should have raised a ProgrammingError") + def CheckClosedCreateFunction(self): con = sqlite.connect(":memory:") con.close() @@ -921,6 +1096,99 @@ def CheckOnConflictReplace(self): self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')]) + +class ClosedBlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckClosedRead(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.read() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedWrite(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.write(b"aaaaaaaaa") + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedSeek(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.seek(10) + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedTell(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.tell() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedClose(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.close() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + +class BlobContextManagerTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckContextExecute(self): + data = b"a" * 100 + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.write(data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) + + def CheckContextCloseBlob(self): + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.seek(10) + try: + blob.close() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def suite(): module_suite = unittest.makeSuite(ModuleTests, "Check") connection_suite = unittest.makeSuite(ConnectionTests, "Check") @@ -931,11 +1199,12 @@ def suite(): closed_con_suite = unittest.makeSuite(ClosedConTests, "Check") closed_cur_suite = unittest.makeSuite(ClosedCurTests, "Check") on_conflict_suite = unittest.makeSuite(SqliteOnConflictTests, "Check") - return unittest.TestSuite(( - module_suite, connection_suite, cursor_suite, thread_suite, - constructor_suite, ext_suite, closed_con_suite, closed_cur_suite, - on_conflict_suite, - )) + blob_suite = unittest.makeSuite(BlobTests, "Check") + closed_blob_suite = unittest.makeSuite(ClosedBlobTests, "Check") + blob_context_manager_suite = unittest.makeSuite(BlobContextManagerTests, "Check") + return unittest.TestSuite((module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, + ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, + blob_suite, closed_blob_suite, blob_context_manager_suite)) def test(): runner = unittest.TextTestRunner() diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c new file mode 100644 index 00000000000000..11da2b0ad5d691 --- /dev/null +++ b/Modules/_sqlite/blob.c @@ -0,0 +1,342 @@ +#include "blob.h" +#include "util.h" + + +int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, + sqlite3_blob *blob) +{ + Py_INCREF(connection); + self->connection = connection; + self->offset = 0; + self->blob = blob; + self->in_weakreflist = NULL; + + if (!pysqlite_check_thread(self->connection)) { + return -1; + } + return 0; +} + +static void remove_blob_from_connection_blob_list(pysqlite_Blob *self) +{ + Py_ssize_t i; + PyObject *item; + + for (i = 0; i < PyList_GET_SIZE(self->connection->blobs); i++) { + item = PyList_GET_ITEM(self->connection->blobs, i); + if (PyWeakref_GetObject(item) == (PyObject *)self) { + PyList_SetSlice(self->connection->blobs, i, i+1, NULL); + break; + } + } +} + +static void _close_blob_inner(pysqlite_Blob* self) +{ + sqlite3_blob *blob; + + /* close the blob */ + blob = self->blob; + self->blob = NULL; + if (blob) { + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + } + + /* remove from connection weaklist */ + remove_blob_from_connection_blob_list(self); + if (self->in_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject*)self); + } +} + +static void pysqlite_blob_dealloc(pysqlite_Blob* self) +{ + _close_blob_inner(self); + Py_XDECREF(self->connection); + Py_TYPE(self)->tp_free((PyObject*)self); +} + + +/* + * Checks if a blob object is usable (i. e. not closed). + * + * 0 => error; 1 => ok + */ +int pysqlite_check_blob(pysqlite_Blob *blob) +{ + + if (!blob->blob) { + PyErr_SetString(pysqlite_ProgrammingError, + "Cannot operate on a closed blob."); + return 0; + } else if (!pysqlite_check_connection(blob->connection) || + !pysqlite_check_thread(blob->connection)) { + return 0; + } else { + return 1; + } +} + + +PyObject* pysqlite_blob_close(pysqlite_Blob *self) +{ + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + _close_blob_inner(self); + Py_RETURN_NONE; +}; + + +static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) +{ + int blob_length; + if (!pysqlite_check_blob(self)) { + return -1; + } + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + return blob_length; +}; + + +PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) +{ + int read_length = -1; + int blob_length = 0; + PyObject *buffer; + char *raw_buffer; + int rc; + + if (!PyArg_ParseTuple(args, "|i", &read_length)) { + return NULL; + } + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + + /* TODO: make this multithreaded and safe! */ + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (read_length < 0) { + /* same as file read. */ + read_length = blob_length; + } + + /* making sure we don't read more then blob size */ + if (read_length > blob_length - self->offset) { + read_length = blob_length - self->offset; + } + + buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (!buffer) { + return NULL; + } + raw_buffer = PyBytes_AS_STRING(buffer); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + Py_DECREF(buffer); + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return NULL; + } + + /* update offset. */ + self->offset += read_length; + + return buffer; +}; + + +PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) +{ + Py_buffer data_buffer; + int rc; + + if (PyObject_GetBuffer(data, &data_buffer, PyBUF_SIMPLE) < 0) { + return NULL; + } + + if (data_buffer.len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "data longer than INT_MAX bytes"); + PyBuffer_Release(&data_buffer); + return NULL; + } + + if (!pysqlite_check_blob(self)) { + PyBuffer_Release(&data_buffer); + return NULL; + } + + /* TODO: throw better error on data bigger then blob. */ + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buffer.buf, + data_buffer.len, self->offset); + Py_END_ALLOW_THREADS + if (rc != SQLITE_OK) { + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyBuffer_Release(&data_buffer); + return NULL; + } + + self->offset += (int)data_buffer.len; + PyBuffer_Release(&data_buffer); + Py_RETURN_NONE; +} + + +PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) +{ + int blob_length, offset, from_what = 0; + + if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { + return NULL; + } + + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + switch (from_what) { + case 0: // relative to blob begin + break; + case 1: // relative to current position + if (offset > INT_MAX - self->offset) { + goto overflow; + } + offset = self->offset + offset; + break; + case 2: // relative to blob end + if (offset > INT_MAX - blob_length) { + goto overflow; + } + offset = blob_length + offset; + break; + default: + return PyErr_Format(PyExc_ValueError, + "from_what should be 0, 1 or 2"); + } + + if (offset < 0 || offset > blob_length) { + return PyErr_Format(PyExc_ValueError, "offset out of blob range"); + } + + self->offset = offset; + Py_RETURN_NONE; + +overflow: + return PyErr_Format(PyExc_OverflowError, "seek offset result in overflow"); +} + + +PyObject* pysqlite_blob_tell(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + return PyLong_FromLong(self->offset); +} + + +PyObject* pysqlite_blob_enter(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_INCREF(self); + return (PyObject *)self; +} + + +PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) +{ + PyObject *res; + if (!pysqlite_check_blob(self)) { + return NULL; + } + + res = pysqlite_blob_close(self); + if (!res) { + return NULL; + } + Py_XDECREF(res); + + Py_RETURN_FALSE; +} + + +static PyMethodDef blob_methods[] = { + {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, + PyDoc_STR("read data from blob")}, + {"write", (PyCFunction)pysqlite_blob_write, METH_O, + PyDoc_STR("write data to blob")}, + {"close", (PyCFunction)pysqlite_blob_close, METH_NOARGS, + PyDoc_STR("close blob")}, + {"seek", (PyCFunction)pysqlite_blob_seek, METH_VARARGS, + PyDoc_STR("change blob current offset")}, + {"tell", (PyCFunction)pysqlite_blob_tell, METH_NOARGS, + PyDoc_STR("return blob current offset")}, + {"__enter__", (PyCFunction)pysqlite_blob_enter, METH_NOARGS, + PyDoc_STR("blob context manager enter")}, + {"__exit__", (PyCFunction)pysqlite_blob_exit, METH_VARARGS, + PyDoc_STR("blob context manager exit")}, + {NULL, NULL} +}; + +static PySequenceMethods blob_sequence_methods = { + (lenfunc)pysqlite_blob_length, /* sq_length */ +}; + + +PyTypeObject pysqlite_BlobType = { + PyVarObject_HEAD_INIT(NULL, 0) + MODULE_NAME ".Blob", + .tp_basicsize = sizeof(pysqlite_Blob), + .tp_dealloc = (destructor)pysqlite_blob_dealloc, + .tp_as_sequence = &blob_sequence_methods, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), + .tp_methods = blob_methods, +}; + +extern int pysqlite_blob_setup_types(void) +{ + pysqlite_BlobType.tp_new = PyType_GenericNew; + return PyType_Ready(&pysqlite_BlobType); +} diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h new file mode 100644 index 00000000000000..204918fd4f01c6 --- /dev/null +++ b/Modules/_sqlite/blob.h @@ -0,0 +1,25 @@ +#ifndef PYSQLITE_BLOB_H +#define PYSQLITE_BLOB_H +#include "Python.h" +#include "sqlite3.h" +#include "connection.h" + +typedef struct +{ + PyObject_HEAD + pysqlite_Connection* connection; + sqlite3_blob *blob; + int offset; + + PyObject* in_weakreflist; /* List of weak references */ +} pysqlite_Blob; + +extern PyTypeObject pysqlite_BlobType; + +int pysqlite_blob_init(pysqlite_Blob* self, pysqlite_Connection* connection, + sqlite3_blob *blob); +PyObject* pysqlite_blob_close(pysqlite_Blob *self); + +int pysqlite_blob_setup_types(void); + +#endif diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 958be7d869794a..b99fcdb24fc25f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -27,6 +27,7 @@ #include "connection.h" #include "statement.h" #include "cursor.h" +#include "blob.h" #include "prepare_protocol.h" #include "util.h" @@ -105,7 +106,8 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject Py_CLEAR(self->statement_cache); Py_CLEAR(self->statements); Py_CLEAR(self->cursors); - + Py_CLEAR(self->blobs); + Py_INCREF(Py_None); Py_XSETREF(self->row_factory, Py_None); @@ -159,10 +161,11 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject self->created_statements = 0; self->created_cursors = 0; - /* Create lists of weak references to statements/cursors */ + /* Create lists of weak references to statements/cursors/blobs */ self->statements = PyList_New(0); self->cursors = PyList_New(0); - if (!self->statements || !self->cursors) { + self->blobs = PyList_New(0); + if (!self->statements || !self->cursors || !self->blobs) { return -1; } @@ -258,6 +261,8 @@ void pysqlite_connection_dealloc(pysqlite_Connection* self) Py_XDECREF(self->collations); Py_XDECREF(self->statements); Py_XDECREF(self->cursors); + Py_XDECREF(self->blobs); + Py_TYPE(self)->tp_free((PyObject*)self); } @@ -327,6 +332,84 @@ PyObject* pysqlite_connection_cursor(pysqlite_Connection* self, PyObject* args, return cursor; } +PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, + PyObject *kwargs) +{ + static char *kwlist[] = {"table", "column", "row", "readonly", + "dbname", NULL, NULL}; + int rc; + const char *dbname = "main", *table, *column; + long long row; + int readonly = 0; + sqlite3_blob *blob; + pysqlite_Blob *pyblob = NULL; + PyObject *weakref; + + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|ps", kwlist, + &table, &column, &row, &readonly, + &dbname)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_open(self->db, dbname, table, column, row, + !readonly, &blob); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK) { + _pysqlite_seterror(self->db, NULL); + return NULL; + } + + pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + if (!pyblob) { + goto error; + } + + rc = pysqlite_blob_init(pyblob, self, blob); + if (rc) { + Py_CLEAR(pyblob); + goto error; + } + + // Add our blob to connection blobs list + weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); + if (!weakref) { + Py_CLEAR(pyblob); + goto error; + } + if (PyList_Append(self->blobs, weakref) != 0) { + Py_CLEAR(weakref); + Py_CLEAR(pyblob); + goto error; + } + Py_DECREF(weakref); + + return (PyObject*)pyblob; + +error: + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + return NULL; +} + +static void pysqlite_close_all_blobs(pysqlite_Connection *self) +{ + int i; + PyObject *weakref; + PyObject *blob; + + for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + weakref = PyList_GET_ITEM(self->blobs, i); + blob = PyWeakref_GetObject(weakref); + if (blob != Py_None) { + pysqlite_blob_close((pysqlite_Blob*)blob); + } + } +} + PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) { int rc; @@ -337,6 +420,8 @@ PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) pysqlite_do_all_statements(self, ACTION_FINALIZE, 1); + pysqlite_close_all_blobs(self); + if (self->db) { rc = SQLITE3_CLOSE(self->db); @@ -1768,6 +1853,8 @@ static PyGetSetDef connection_getset[] = { static PyMethodDef connection_methods[] = { {"cursor", (PyCFunction)(void(*)(void))pysqlite_connection_cursor, METH_VARARGS|METH_KEYWORDS, PyDoc_STR("Return a cursor for the connection.")}, + {"open_blob", (PyCFunction)pysqlite_connection_blob, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("return a blob object")}, {"close", (PyCFunction)pysqlite_connection_close, METH_NOARGS, PyDoc_STR("Closes the connection.")}, {"commit", (PyCFunction)pysqlite_connection_commit, METH_NOARGS, diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 206085e00a00c7..708a199087c002 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -66,9 +66,10 @@ typedef struct pysqlite_Cache* statement_cache; - /* Lists of weak references to statements and cursors used within this connection */ + /* Lists of weak references to statements, blobs and cursors used within this connection */ PyObject* statements; PyObject* cursors; + PyObject* blobs; /* Counters for how many statements/cursors were created in the connection. May be * reset to 0 at certain intervals */ diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 71d951ee887e47..f138e02357753f 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -28,6 +28,7 @@ #include "prepare_protocol.h" #include "microprotocols.h" #include "row.h" +#include "blob.h" #if SQLITE_VERSION_NUMBER >= 3003003 #define HAVE_SHARED_CACHE @@ -368,7 +369,8 @@ PyMODINIT_FUNC PyInit__sqlite3(void) (pysqlite_connection_setup_types() < 0) || (pysqlite_cache_setup_types() < 0) || (pysqlite_statement_setup_types() < 0) || - (pysqlite_prepare_protocol_setup_types() < 0) + (pysqlite_prepare_protocol_setup_types() < 0) || + (pysqlite_blob_setup_types() < 0) ) { Py_XDECREF(module); return NULL; diff --git a/setup.py b/setup.py index 21a5a58981fc15..8c70025489e181 100644 --- a/setup.py +++ b/setup.py @@ -1523,7 +1523,8 @@ def detect_sqlite(self): '_sqlite/prepare_protocol.c', '_sqlite/row.c', '_sqlite/statement.c', - '_sqlite/util.c', ] + '_sqlite/util.c', + '_sqlite/blob.c' ] sqlite_defines = [] if not MS_WINDOWS: From 3fe91083c9b015ec326279a37e8c297e49747d61 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:11:32 +0200 Subject: [PATCH 02/18] Note that blob size cannot be changed using the blob object --- Doc/library/sqlite3.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ceb92b3fe90fab..e464192e136c13 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -309,6 +309,11 @@ Connection Objects When *readonly* is :const:`True` the BLOB is opened with read permissions. Otherwise the BLOB has read and write permissions. + .. note:: + + The BLOB size cannot be changed using the :class:`Blob` class. Use + `zeroblob` to create the blob in the wanted size in advance. + .. versionadded:: 3.7 .. method:: commit() From 865c1c84be443770f489138dd718f9087a080ac4 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:22:03 +0200 Subject: [PATCH 03/18] Use assertRaises in tests --- Lib/sqlite3/test/dbapi.py | 119 ++++++-------------------------------- 1 file changed, 17 insertions(+), 102 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 85db83ee51f246..8b2fc70bce3abf 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -546,22 +546,12 @@ def CheckSeekFromBlobEnd(self): self.assertEqual(self.blob.tell(), 90) def CheckBlobSeekOverBlobSize(self): - try: + with self.assertRaises(ValueError): self.blob.seek(1000) - self.fail("should have raised a ValueError") - except ValueError: - pass - except Exception: - self.fail("should have raised a ValueError") def CheckBlobSeekUnderBlobSize(self): - try: + with self.assertRaises(ValueError): self.blob.seek(-10) - self.fail("should have raised a ValueError") - except ValueError: - pass - except Exception: - self.fail("should have raised a ValueError") def CheckBlobRead(self): self.assertEqual(self.blob.read(), self.blob_data) @@ -594,81 +584,41 @@ def CheckBlobWriteAdvanceOffset(self): self.assertEqual(self.blob.tell(), 50) def CheckBlobWriteMoreThenBlobSize(self): - try: + with self.assertRaises(sqlite.OperationalError): self.blob.write(b"a" * 1000) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobReadAfterRowChange(self): self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") - try: + with self.assertRaises(sqlite.OperationalError): self.blob.read() - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobWriteAfterRowChange(self): self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") - try: + with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobWriteWhenReadOnly(self): read_only_blob = \ self.cx.open_blob("test", "blob_col", 1, readonly=True) - try: + with self.assertRaises(sqlite.OperationalError): read_only_blob.write(b"aaa") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") read_only_blob.close() def CheckBlobOpenWithBadDb(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadTable(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("notexisintg", "blob_col", 1) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadColumn(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "notexisting", 1) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadRow(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 2) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") @unittest.skipUnless(threading, 'This test requires threading.') @@ -935,13 +885,8 @@ def CheckClosedBlobRead(self): con.execute("insert into test(blob_col) values (zeroblob(100))") blob = con.open_blob("test", "blob_col", 1) con.close() - try: + with self.assertRaises(sqlite.ProgrammingError): blob.read() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except: - self.fail("Should have raised a ProgrammingError") def CheckClosedCreateFunction(self): con = sqlite.connect(":memory:") @@ -1109,57 +1054,32 @@ def tearDown(self): def CheckClosedRead(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.read() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedWrite(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.write(b"aaaaaaaaa") - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedSeek(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.seek(10) - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedTell(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.tell() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedClose(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.close() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") class BlobContextManagerTests(unittest.TestCase): @@ -1180,13 +1100,8 @@ def CheckContextExecute(self): def CheckContextCloseBlob(self): with self.cx.open_blob("test", "blob_col", 1) as blob: blob.seek(10) - try: + with self.assertRaises(sqlite.ProgrammingError): blob.close() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def suite(): From 788fd54d56e52df55fdfff33b481b032beeb449b Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:38:43 +0200 Subject: [PATCH 04/18] Fix doc error --- Doc/library/sqlite3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e464192e136c13..6afef818e4ad70 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -312,7 +312,7 @@ Connection Objects .. note:: The BLOB size cannot be changed using the :class:`Blob` class. Use - `zeroblob` to create the blob in the wanted size in advance. + ``zeroblob`` to create the blob in the wanted size in advance. .. versionadded:: 3.7 From a1361e5ed71cbd1ef461a5eb09dd695a9e66619d Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:26:39 +0200 Subject: [PATCH 05/18] blob support sequence protocol --- Lib/sqlite3/test/dbapi.py | 56 ++++++ Modules/_sqlite/blob.c | 396 ++++++++++++++++++++++++++++++++++---- 2 files changed, 416 insertions(+), 36 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 8b2fc70bce3abf..24122a0cf58062 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -620,6 +620,62 @@ def CheckBlobOpenWithBadRow(self): with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 2) + def CheckBlobGetItem(self): + self.assertEqual(self.blob[5], b"a") + + def CheckBlobGetItemIndexOutOfRange(self): + with self.assertRaises(IndexError): + self.blob[105] + with self.assertRaises(IndexError): + self.blob[-105] + + def CheckBlobGetItemNegativeIndex(self): + self.assertEqual(self.blob[-5], b"a") + + def CheckBlobGetItemInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[b"a"] + + def CheckBlobGetSlice(self): + self.assertEqual(self.blob[5:10], b"aaaaa") + + def CheckBlobGetSliceNegativeIndex(self): + self.assertEqual(self.blob[5:-5], self.blob_data[5:-5]) + + def CheckBlobGetSliceInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[5:b"a"] + + def CheckBlobGetSliceWithSkip(self): + self.blob.write(b"abcdefghij") + self.assertEqual(self.blob[0:10:2], b"acegi") + + def CheckBlobSetItem(self): + self.blob[0] = b"b" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"b" + self.blob_data[1:]) + + def CheckBlobSetSlice(self): + self.blob[0:5] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bbbbb" + self.blob_data[5:]) + + def CheckBlobSetSliceWithSkip(self): + self.blob[0:10:2] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bababababa" + self.blob_data[10:]) + + def CheckBlobGetEmptySlice(self): + self.assertEqual(self.blob[5:5], b"") + + def CheckBlobSetSliceWrongLength(self): + with self.assertRaises(IndexError): + self.blob[5:10] = b"a" + + def CheckBlobConcatNotSupported(self): + with self.assertRaises(SystemError): + self.blob + self.blob + + def CheckBlobRepeateNotSupported(self): + with self.assertRaises(SystemError): + self.blob * 5 @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 11da2b0ad5d691..333c65425a1f0f 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -105,14 +105,43 @@ static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) return blob_length; }; +static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) +{ + PyObject *buffer; + char *raw_buffer; + int rc; + + buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (!buffer) { + return NULL; + } + raw_buffer = PyBytes_AS_STRING(buffer); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + Py_DECREF(buffer); + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return NULL; + } + return buffer; +} + PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) { int read_length = -1; int blob_length = 0; PyObject *buffer; - char *raw_buffer; - int rc; if (!PyArg_ParseTuple(args, "|i", &read_length)) { return NULL; @@ -138,34 +167,36 @@ PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) read_length = blob_length - self->offset; } - buffer = PyBytes_FromStringAndSize(NULL, read_length); - if (!buffer) { - return NULL; + buffer = inner_read(self, read_length, self->offset); + + if (buffer != NULL) { + /* update offset on sucess. */ + self->offset += read_length; } - raw_buffer = PyBytes_AS_STRING(buffer); + + return buffer; +}; + +static int write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +{ + int rc; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + rc = sqlite3_blob_write(self->blob, buf, len, offset); Py_END_ALLOW_THREADS - - if (rc != SQLITE_OK){ - Py_DECREF(buffer); + if (rc != SQLITE_OK) { /* For some reason after modifying blob the - error is not set on the connection db. */ + error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(pysqlite_OperationalError, "Cannot operate on modified blob"); } else { _pysqlite_seterror(self->connection->db, NULL); } - return NULL; + return -1; } - - /* update offset. */ - self->offset += read_length; - - return buffer; -}; + return 0; +} PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) @@ -191,26 +222,15 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) /* TODO: throw better error on data bigger then blob. */ - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_write(self->blob, data_buffer.buf, - data_buffer.len, self->offset); - Py_END_ALLOW_THREADS - if (rc != SQLITE_OK) { - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(pysqlite_OperationalError, - "Cannot operate on modified blob"); - } else { - _pysqlite_seterror(self->connection->db, NULL); - } + rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); + + if (rc == 0) { + self->offset += (int)data_buffer.len; + Py_RETURN_NONE; + } else { PyBuffer_Release(&data_buffer); return NULL; } - - self->offset += (int)data_buffer.len; - PyBuffer_Release(&data_buffer); - Py_RETURN_NONE; } @@ -300,6 +320,300 @@ PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) Py_RETURN_FALSE; } +static PyObject* pysqlite_blob_concat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support concatenation"); + } + return NULL; +} + +static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support repeat operation"); + } + return NULL; +} + +static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) +{ + int blob_length = 0; + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return NULL; + } + + return inner_read(self, 1, i); +} + +static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) +{ + int blob_length = 0; + const char *buf; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(v) && PyBytes_Size(v)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(v); + return write_inner(self, buf, 1, i); +} + + +static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) +{ + int blob_length = 0; + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if (i < 0) + i += blob_length; + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return NULL; + } + // TODO: I am not sure... + return inner_read(self, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + + if (PySlice_GetIndicesEx(item, blob_length, + &start, &stop, &step, &slicelen) < 0) { + return NULL; + } + + if (slicelen <= 0) + return PyBytes_FromStringAndSize("", 0); + else if (step == 1) + return inner_read(self, slicelen, start); + else { + char *result_buf = (char *)PyMem_Malloc(slicelen); + char *data_buff = NULL; + Py_ssize_t cur, i; + PyObject *result; + int rc; + + if (result_buf == NULL) + return PyErr_NoMemory(); + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyMem_Free(result_buf); + return PyErr_NoMemory(); + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return NULL; + } + + for (cur = 0, i = 0; i < slicelen; + cur += step, i++) { + result_buf[i] = data_buff[cur]; + } + result = PyBytes_FromStringAndSize(result_buf, + slicelen); + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return result; + } + } + else { + PyErr_SetString(PyExc_TypeError, + "Blob indices must be integers"); + return NULL; + } +} + + +static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +{ + int blob_length = 0; + int rc; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + const char *buf; + + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += blob_length; + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(value); + return write_inner(self, buf, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + Py_buffer vbuf; + + if (PySlice_GetIndicesEx(item, + blob_length, &start, &stop, + &step, &slicelen) < 0) { + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support slice deletion"); + return -1; + } + if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) + return -1; + if (vbuf.len != slicelen) { + PyErr_SetString(PyExc_IndexError, + "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; + } + + if (slicelen == 0) { + } + else if (step == 1) { + rc = write_inner(self, vbuf.buf, slicelen, start); + } + else { + Py_ssize_t cur, i; + char *data_buff; + + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyErr_NoMemory(); + return -1; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + + for (cur = 0, i = 0; + i < slicelen; + cur += step, i++) + { + data_buff[cur] = ((char *)vbuf.buf)[i]; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + rc = 0; + + } + PyBuffer_Release(&vbuf); + return rc; + } + else { + PyErr_SetString(PyExc_TypeError, + "mmap indices must be integer"); + return -1; + } +} + static PyMethodDef blob_methods[] = { {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, @@ -320,9 +634,18 @@ static PyMethodDef blob_methods[] = { }; static PySequenceMethods blob_sequence_methods = { - (lenfunc)pysqlite_blob_length, /* sq_length */ + .sq_length = (lenfunc)pysqlite_blob_length, + .sq_concat = (binaryfunc)pysqlite_blob_concat, + .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, + .sq_item = (ssizeargfunc)pysqlite_blob_item, + .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, }; +static PyMappingMethods blob_mapping_methods = { + (lenfunc)pysqlite_blob_length, + (binaryfunc)pysqlite_blob_subscript, + (objobjargproc)pysqlite_blob_ass_subscript, +}; PyTypeObject pysqlite_BlobType = { PyVarObject_HEAD_INIT(NULL, 0) @@ -330,6 +653,7 @@ PyTypeObject pysqlite_BlobType = { .tp_basicsize = sizeof(pysqlite_Blob), .tp_dealloc = (destructor)pysqlite_blob_dealloc, .tp_as_sequence = &blob_sequence_methods, + .tp_as_mapping = &blob_mapping_methods, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), .tp_methods = blob_methods, From 5aedfba9a1541473e72faaaaabf3d63e0b972ad6 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:34:59 +0200 Subject: [PATCH 06/18] Calculate blob length once at creation --- Modules/_sqlite/blob.c | 73 ++++++++++++------------------------------ Modules/_sqlite/blob.h | 1 + 2 files changed, 21 insertions(+), 53 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 333c65425a1f0f..abe8cac1b271bc 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,6 +11,10 @@ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, self->blob = blob; self->in_weakreflist = NULL; + Py_BEGIN_ALLOW_THREADS + self->length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + if (!pysqlite_check_thread(self->connection)) { return -1; } @@ -94,15 +98,11 @@ PyObject* pysqlite_blob_close(pysqlite_Blob *self) static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) { - int blob_length; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - return blob_length; + return self->length; }; static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) @@ -140,7 +140,6 @@ static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) { int read_length = -1; - int blob_length = 0; PyObject *buffer; if (!PyArg_ParseTuple(args, "|i", &read_length)) { @@ -151,20 +150,14 @@ PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) return NULL; } - - /* TODO: make this multithreaded and safe! */ - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (read_length < 0) { /* same as file read. */ - read_length = blob_length; + read_length = self->length; } /* making sure we don't read more then blob size */ - if (read_length > blob_length - self->offset) { - read_length = blob_length - self->offset; + if (read_length > self->length - self->offset) { + read_length = self->length - self->offset; } buffer = inner_read(self, read_length, self->offset); @@ -236,7 +229,7 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) { - int blob_length, offset, from_what = 0; + int offset, from_what = 0; if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { return NULL; @@ -247,10 +240,6 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - switch (from_what) { case 0: // relative to blob begin break; @@ -261,17 +250,17 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->offset + offset; break; case 2: // relative to blob end - if (offset > INT_MAX - blob_length) { + if (offset > INT_MAX - self->length) { goto overflow; } - offset = blob_length + offset; + offset = self->length + offset; break; default: return PyErr_Format(PyExc_ValueError, "from_what should be 0, 1 or 2"); } - if (offset < 0 || offset > blob_length) { + if (offset < 0 || offset > self->length) { return PyErr_Format(PyExc_ValueError, "offset out of blob range"); } @@ -340,17 +329,11 @@ static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) { - int blob_length = 0; - if (!pysqlite_check_blob(self)) { return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (i < 0 || i >= blob_length) { + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } @@ -360,18 +343,13 @@ static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) { - int blob_length = 0; const char *buf; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (i < 0 || i >= blob_length) { + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } @@ -393,23 +371,17 @@ static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) { - int blob_length = 0; - if (!pysqlite_check_blob(self)) { return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) - i += blob_length; - if (i < 0 || i >= blob_length) { + i += self->length; + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; @@ -420,7 +392,7 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, blob_length, + if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, &slicelen) < 0) { return NULL; } @@ -484,17 +456,12 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) { - int blob_length = 0; int rc; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); const char *buf; @@ -502,8 +469,8 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) - i += blob_length; - if (i < 0 || i >= blob_length) { + i += self->length; + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; @@ -527,7 +494,7 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb Py_buffer vbuf; if (PySlice_GetIndicesEx(item, - blob_length, &start, &stop, + self->length, &start, &stop, &step, &slicelen) < 0) { return -1; } diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 204918fd4f01c6..649f09e5ecca24 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -10,6 +10,7 @@ typedef struct pysqlite_Connection* connection; sqlite3_blob *blob; int offset; + int length; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Blob; From db6ef327457f9eb56e448c50309b714322212eed Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:41:20 +0200 Subject: [PATCH 07/18] Add initial Doc for sequence protocol --- Doc/library/sqlite3.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 6afef818e4ad70..068e15bf2534b9 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -878,7 +878,9 @@ Blob Objects .. class:: Blob A :class:`Blob` instance can read and write the data in the - :abbr:`BLOB (Binary Large OBject)`. + :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both + the file and sequence protocol. For example You can read data from the + :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. .. method:: Blob.close() From 219f4cb4a1b3db1783f4e9c272cd3e6de4028908 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:04:22 +0300 Subject: [PATCH 08/18] Don't support blob operation --- Lib/sqlite3/test/dbapi.py | 4 ++++ Modules/_sqlite/blob.c | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 24122a0cf58062..d769d62446b2b7 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -677,6 +677,10 @@ def CheckBlobRepeateNotSupported(self): with self.assertRaises(SystemError): self.blob * 5 + def CheckBlobContainsNotSupported(self): + with self.assertRaises(SystemError): + b"aaaaa" in self.blob + @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index abe8cac1b271bc..7b2432491450a4 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -327,6 +327,15 @@ static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) return NULL; } +static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support cotains operation"); + } + return -1; +} + static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) { if (!pysqlite_check_blob(self)) { @@ -606,6 +615,7 @@ static PySequenceMethods blob_sequence_methods = { .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, .sq_item = (ssizeargfunc)pysqlite_blob_item, .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, + .sq_contains = (objobjproc)pysqlite_blob_contains, }; static PyMappingMethods blob_mapping_methods = { From 6dafe0e7b5d651f373304f39961c5a5552e05531 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:16:26 +0300 Subject: [PATCH 09/18] move news entry to blurb --- .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst new file mode 100644 index 00000000000000..4f7b2fefed8039 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -0,0 +1,4 @@ +The :class:`sqlite3.Connection` now has the +:meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` +allows incremental I/O operations to blobs. (Contributed by Aviv Palivoda in +:issue:`24905`) From ffac9013a89c68561b51443585a799aacad3b6e7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:32:15 +0300 Subject: [PATCH 10/18] Add blob to PCBuild --- PCbuild/_sqlite3.vcxproj | 2 ++ PCbuild/_sqlite3.vcxproj.filters | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 7e0062692b8f83..55f46c963b1e7d 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -107,6 +107,7 @@ + @@ -118,6 +119,7 @@ + diff --git a/PCbuild/_sqlite3.vcxproj.filters b/PCbuild/_sqlite3.vcxproj.filters index 51830f6a4451a4..8d26c9ab6eb43f 100644 --- a/PCbuild/_sqlite3.vcxproj.filters +++ b/PCbuild/_sqlite3.vcxproj.filters @@ -39,6 +39,9 @@ Header Files + + Header Files + @@ -68,6 +71,9 @@ Source Files + + Source Files + From 3475fa1c422a7418dc1a7765ad33479307afa531 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 15:30:45 +0300 Subject: [PATCH 11/18] Update version --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 068e15bf2534b9..9b6d0472089063 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -314,7 +314,7 @@ Connection Objects The BLOB size cannot be changed using the :class:`Blob` class. Use ``zeroblob`` to create the blob in the wanted size in advance. - .. versionadded:: 3.7 + .. versionadded:: 3.8 .. method:: commit() @@ -873,7 +873,7 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.7 +.. versionadded:: 3.8 .. class:: Blob From f6015ff3b39343898066e70c879d17dc5a78f89d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 15:32:14 +0300 Subject: [PATCH 12/18] Fix memory leak --- Modules/_sqlite/blob.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7b2432491450a4..d560563d3e766d 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -219,6 +219,7 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) if (rc == 0) { self->offset += (int)data_buffer.len; + PyBuffer_Release(&data_buffer); Py_RETURN_NONE; } else { PyBuffer_Release(&data_buffer); From 01e526ca5c94b480380919a79e320037bc7f65f0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 29 Jul 2020 19:40:23 +0300 Subject: [PATCH 13/18] Update version --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9b6d0472089063..841b2be4fdacf8 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -314,7 +314,7 @@ Connection Objects The BLOB size cannot be changed using the :class:`Blob` class. Use ``zeroblob`` to create the blob in the wanted size in advance. - .. versionadded:: 3.8 + .. versionadded:: 3.10 .. method:: commit() @@ -873,7 +873,7 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.8 +.. versionadded:: 3.10 .. class:: Blob From 354bebf2dc561a6f079fc430edf8d9eb7cab6496 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:01:26 +0300 Subject: [PATCH 14/18] Fix CR comments in documentation and testing --- Doc/includes/sqlite3/blob.py | 6 +++--- Doc/includes/sqlite3/blob_with.py | 6 +++--- Doc/library/sqlite3.rst | 5 +++-- Lib/sqlite3/test/dbapi.py | 8 +++++--- .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index b8d2c78d1a365d..18514876af6191 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -6,8 +6,8 @@ con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle blob = con.open_blob("test", "blob_col", 1, 1) -blob.write(b"a" * 5) -blob.write(b"b" * 5) +blob.write(b"Hello") +blob.write(b"World") blob.seek(0) -print(blob.read()) # will print b"aaaaabbbbb" +print(blob.read()) # will print b"HelloWorld" blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index 624b680591901a..f220518c09a1ab 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -6,7 +6,7 @@ con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle with con.open_blob("test", "blob_col", 1, 1) as blob: - blob.write(b"a" * 5) - blob.write(b"b" * 5) + blob.write(b"Hello") + blob.write(b"World") blob.seek(0) - print(blob.read()) # will print b"aaaaabbbbb" + print(blob.read()) # will print b"HelloWorld" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 841b2be4fdacf8..b754481ad48757 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -879,8 +879,9 @@ Blob Objects A :class:`Blob` instance can read and write the data in the :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both - the file and sequence protocol. For example You can read data from the + the file and sequence protocol. For example, you can read data from the :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. + You can call ``len(obj)`` to get size of the BLOB. .. method:: Blob.close() @@ -911,7 +912,7 @@ Blob Objects Return the current offset of the BLOB. - .. method:: Blob.seek(offset, [whence]) + .. method:: Blob.seek(offset, whence=os.SEEK_SET) Set the BLOB offset. The *whence* argument is optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index d769d62446b2b7..a611feb56414eb 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -1177,9 +1177,11 @@ def suite(): blob_suite = unittest.makeSuite(BlobTests, "Check") closed_blob_suite = unittest.makeSuite(ClosedBlobTests, "Check") blob_context_manager_suite = unittest.makeSuite(BlobContextManagerTests, "Check") - return unittest.TestSuite((module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, - ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, - blob_suite, closed_blob_suite, blob_context_manager_suite)) + return unittest.TestSuite(( + module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, + ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, + blob_suite, closed_blob_suite, blob_context_manager_suite, + )) def test(): runner = unittest.TextTestRunner() diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst index 4f7b2fefed8039..c7d2405d89539e 100644 --- a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -1,4 +1,4 @@ The :class:`sqlite3.Connection` now has the :meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` -allows incremental I/O operations to blobs. (Contributed by Aviv Palivoda in +allows incremental I/O operations to blobs. (Patch by Aviv Palivoda in :issue:`24905`) From 9709456b864e48e1cc3a3e838b311806a9ca6acb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:08:58 +0300 Subject: [PATCH 15/18] Fix CR comments in code --- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/connection.h | 2 +- setup.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index b99fcdb24fc25f..7e112b7fdb705a 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -336,7 +336,7 @@ PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = {"table", "column", "row", "readonly", - "dbname", NULL, NULL}; + "dbname", NULL}; int rc; const char *dbname = "main", *table, *column; long long row; diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 708a199087c002..52dc27c8adff2d 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -66,7 +66,7 @@ typedef struct pysqlite_Cache* statement_cache; - /* Lists of weak references to statements, blobs and cursors used within this connection */ + /* Lists of weak references to statements, cursors and blobs used within this connection */ PyObject* statements; PyObject* cursors; PyObject* blobs; diff --git a/setup.py b/setup.py index 8c70025489e181..dd6b3509247fd8 100644 --- a/setup.py +++ b/setup.py @@ -1524,7 +1524,8 @@ def detect_sqlite(self): '_sqlite/row.c', '_sqlite/statement.c', '_sqlite/util.c', - '_sqlite/blob.c' ] + '_sqlite/blob.c', + ] sqlite_defines = [] if not MS_WINDOWS: From e6e5099d5a5d9a4e69317208a4a463922a903fd5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:23:21 +0300 Subject: [PATCH 16/18] Make readonly and dbname keyword arguements only --- Doc/includes/sqlite3/blob.py | 2 +- Doc/includes/sqlite3/blob_with.py | 2 +- Doc/library/sqlite3.rst | 2 +- Modules/_sqlite/connection.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index 18514876af6191..afd7812a8b3af9 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -5,7 +5,7 @@ con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle -blob = con.open_blob("test", "blob_col", 1, 1) +blob = con.open_blob("test", "blob_col", 1) blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index f220518c09a1ab..fdca9fbc638ea2 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -5,7 +5,7 @@ con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle -with con.open_blob("test", "blob_col", 1, 1) as blob: +with con.open_blob("test", "blob_col", 1) as blob: blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index b754481ad48757..827c2e0588611a 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -301,7 +301,7 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. - .. method:: open_blob(table, column, row, readonly=False, dbname="main") + .. method:: open_blob(table, column, row, *, readonly=False, dbname="main") On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 7e112b7fdb705a..1d64bae7ca98f6 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -346,7 +346,7 @@ PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, PyObject *weakref; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|ps", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|$ps", kwlist, &table, &column, &row, &readonly, &dbname)) { return NULL; From e9a80800f0883de7e3b9fb34ca1b8172fd745460 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:32:47 +0300 Subject: [PATCH 17/18] Fix more CR comments --- Modules/_sqlite/blob.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index d560563d3e766d..f222d5cd477f22 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -257,19 +257,19 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->length + offset; break; default: - return PyErr_Format(PyExc_ValueError, + return PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); } if (offset < 0 || offset > self->length) { - return PyErr_Format(PyExc_ValueError, "offset out of blob range"); + return PyErr_SetString(PyExc_ValueError, "offset out of blob range"); } self->offset = offset; Py_RETURN_NONE; overflow: - return PyErr_Format(PyExc_OverflowError, "seek offset result in overflow"); + return PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); } @@ -332,7 +332,7 @@ static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) { if (pysqlite_check_blob(self)) { PyErr_SetString(PyExc_SystemError, - "Blob don't support cotains operation"); + "Blob don't support contains operation"); } return -1; } @@ -407,11 +407,11 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) return NULL; } - if (slicelen <= 0) + if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); - else if (step == 1) + } else if (step == 1) { return inner_read(self, slicelen, start); - else { + } else { char *result_buf = (char *)PyMem_Malloc(slicelen); char *data_buff = NULL; Py_ssize_t cur, i; @@ -586,7 +586,7 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb } else { PyErr_SetString(PyExc_TypeError, - "mmap indices must be integer"); + "Blob indices must be integer"); return -1; } } From d4fb1b5afd50091d99ecaa4022df109626d7325e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:57:07 +0300 Subject: [PATCH 18/18] Add more indicative error on write bigger then blob length --- Lib/sqlite3/test/dbapi.py | 2 +- Modules/_sqlite/blob.c | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index a611feb56414eb..fd88faa7765e02 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -584,7 +584,7 @@ def CheckBlobWriteAdvanceOffset(self): self.assertEqual(self.blob.tell(), 50) def CheckBlobWriteMoreThenBlobSize(self): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaises(ValueError): self.blob.write(b"a" * 1000) def CheckBlobReadAfterRowChange(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f222d5cd477f22..49e664631e2774 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -208,12 +208,17 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) return NULL; } - if (!pysqlite_check_blob(self)) { + if (data_buffer.len > self->length - self->offset) { + PyErr_SetString(PyExc_ValueError, + "data longer than blob length"); PyBuffer_Release(&data_buffer); return NULL; } - /* TODO: throw better error on data bigger then blob. */ + if (!pysqlite_check_blob(self)) { + PyBuffer_Release(&data_buffer); + return NULL; + } rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); @@ -257,19 +262,22 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->length + offset; break; default: - return PyErr_SetString(PyExc_ValueError, + PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); + return NULL; } if (offset < 0 || offset > self->length) { - return PyErr_SetString(PyExc_ValueError, "offset out of blob range"); + PyErr_SetString(PyExc_ValueError, "offset out of blob range"); + return NULL; } self->offset = offset; Py_RETURN_NONE; overflow: - return PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); + PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); + return NULL; } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy