From 3a2b2aabb85fff3f143398431ec7658a3db8186f Mon Sep 17 00:00:00 2001 From: Matthias Fetzer Date: Mon, 15 May 2023 20:15:39 +0200 Subject: [PATCH 1/5] Allow to turn off column store --- src/crate/client/sqlalchemy/compiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index 7e6dad7d..69493abc 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -128,6 +128,9 @@ def get_column_specification(self, column, **kwargs): colspec += " INDEX OFF" + if column.dialect_options['crate'].get('columnstore') is False: + colspec += " STORAGE WITH (columnstore = false)" + return colspec def visit_computed_column(self, generated): From b70f90c2dbed25db67f7114ad4c6619e651edcfb Mon Sep 17 00:00:00 2001 From: Matthias Fetzer Date: Mon, 29 May 2023 11:08:26 +0200 Subject: [PATCH 2/5] Columnstore: Adding tests and error handling for non TEXT columns --- src/crate/client/sqlalchemy/compiler.py | 6 +++ .../sqlalchemy/tests/create_table_test.py | 37 ++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index 69493abc..efa88d17 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -25,6 +25,7 @@ import sqlalchemy as sa from sqlalchemy.dialects.postgresql.base import PGCompiler from sqlalchemy.sql import compiler +from sqlalchemy.types import String from .types import MutableDict, _Craty, Geopoint, Geoshape from .sa_version import SA_VERSION, SA_1_4 @@ -129,6 +130,11 @@ def get_column_specification(self, column, **kwargs): colspec += " INDEX OFF" if column.dialect_options['crate'].get('columnstore') is False: + if not isinstance(column.type, (String, )): + raise sa.exc.CompileError( + "Controlling the columnstore is only allowed for STRING columns" + ) + colspec += " STORAGE WITH (columnstore = false)" return colspec diff --git a/src/crate/client/sqlalchemy/tests/create_table_test.py b/src/crate/client/sqlalchemy/tests/create_table_test.py index 7eca2628..91f54bca 100644 --- a/src/crate/client/sqlalchemy/tests/create_table_test.py +++ b/src/crate/client/sqlalchemy/tests/create_table_test.py @@ -20,6 +20,7 @@ # software solely pursuant to the terms of the relevant commercial agreement. import sqlalchemy as sa + try: from sqlalchemy.orm import declarative_base except ImportError: @@ -31,7 +32,6 @@ from unittest import TestCase from unittest.mock import patch, MagicMock - fake_cursor = MagicMock(name='fake_cursor') FakeCursor = MagicMock(name='FakeCursor', spec=Cursor) FakeCursor.return_value = fake_cursor @@ -77,6 +77,7 @@ class DummyTable(self.Base): __tablename__ = 'dummy' pk = sa.Column(sa.String, primary_key=True) obj_col = sa.Column(Object) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE dummy (\n\tpk STRING NOT NULL, \n\tobj_col OBJECT, ' @@ -91,6 +92,7 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -105,6 +107,7 @@ class DummyTable(self.Base): __tablename__ = 't' ts = sa.Column(sa.BigInteger, primary_key=True) p = sa.Column(sa.BigInteger, sa.Computed("date_trunc('day', ts)")) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -119,6 +122,7 @@ class DummyTable(self.Base): __tablename__ = 't' ts = sa.Column(sa.BigInteger, primary_key=True) p = sa.Column(sa.BigInteger, sa.Computed("date_trunc('day', ts)", persisted=False)) + with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) @@ -131,6 +135,7 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -166,6 +171,7 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String, primary_key=True) + self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -207,6 +213,7 @@ def test_column_pk_nullable(self): class DummyTable(self.Base): __tablename__ = 't' pk = sa.Column(sa.String, primary_key=True, nullable=True) + with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) @@ -230,5 +237,33 @@ class DummyTable(self.Base): __tablename__ = 't' pk = sa.Column(sa.String, primary_key=True) a = sa.Column(Geopoint, crate_index=False) + + with self.assertRaises(sa.exc.CompileError): + self.Base.metadata.create_all(bind=self.engine) + + def test_text_column_without_columnstore(self): + class DummyTable(self.Base): + __tablename__ = 't' + pk = sa.Column(sa.String, primary_key=True) + a = sa.Column(sa.String, crate_columnstore=False) + b = sa.Column(sa.String, crate_columnstore=True) + c = sa.Column(sa.String) + + self.Base.metadata.create_all(bind=self.engine) + + fake_cursor.execute.assert_called_with( + ('\nCREATE TABLE t (\n\t' + 'pk STRING NOT NULL, \n\t' + 'a STRING STORAGE WITH (columnstore = false), \n\t' + 'b STRING, \n\t' + 'c STRING, \n\t' + 'PRIMARY KEY (pk)\n)\n\n'), ()) + + def test_non_text_column_without_columnstore(self): + class DummyTable(self.Base): + __tablename__ = 't' + pk = sa.Column(sa.String, primary_key=True) + a = sa.Column(sa.Integer, crate_columnstore=False) + with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) From da8cae3e2ff236ab98f655b49d4f07d9381504c1 Mon Sep 17 00:00:00 2001 From: Matthias Fetzer Date: Mon, 29 May 2023 11:12:45 +0200 Subject: [PATCH 3/5] Columnstore: Adding docs for controlling the columnstore --- docs/sqlalchemy.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sqlalchemy.rst b/docs/sqlalchemy.rst index 00bf5d00..b16f2abf 100644 --- a/docs/sqlalchemy.rst +++ b/docs/sqlalchemy.rst @@ -205,6 +205,7 @@ system `: ... more_details = sa.Column(types.ObjectArray) ... name_ft = sa.Column(sa.String) ... quote_ft = sa.Column(sa.String) + ... even_more_details = sa.Column(sa.String, crate_columnstore=False) ... ... __mapper_args__ = { ... 'exclude_properties': ['name_ft', 'quote_ft'] @@ -220,6 +221,7 @@ In this example, we: - Use standard SQLAlchemy types for the ``id``, ``name``, and ``quote`` columns - Use ``nullable=False`` to define a ``NOT NULL`` constraint - Disable indexing of the ``name`` column using ``crate_index=False`` +- Disable the columnstore of the ``even_more_details`` column using ``crate_columnstore=False`` - Define a computed column ``name_normalized`` (based on ``name``) that translates into a generated column - Use the `Object`_ extension type for the ``details`` column From b839e114420bdbd424564030bd133df2f1952672 Mon Sep 17 00:00:00 2001 From: Matthias Fetzer Date: Mon, 29 May 2023 11:38:21 +0200 Subject: [PATCH 4/5] Columnstore: Revert (auto) reformatting --- src/crate/client/sqlalchemy/tests/create_table_test.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/crate/client/sqlalchemy/tests/create_table_test.py b/src/crate/client/sqlalchemy/tests/create_table_test.py index 91f54bca..f876655d 100644 --- a/src/crate/client/sqlalchemy/tests/create_table_test.py +++ b/src/crate/client/sqlalchemy/tests/create_table_test.py @@ -20,7 +20,6 @@ # software solely pursuant to the terms of the relevant commercial agreement. import sqlalchemy as sa - try: from sqlalchemy.orm import declarative_base except ImportError: @@ -32,6 +31,7 @@ from unittest import TestCase from unittest.mock import patch, MagicMock + fake_cursor = MagicMock(name='fake_cursor') FakeCursor = MagicMock(name='FakeCursor', spec=Cursor) FakeCursor.return_value = fake_cursor @@ -77,7 +77,6 @@ class DummyTable(self.Base): __tablename__ = 'dummy' pk = sa.Column(sa.String, primary_key=True) obj_col = sa.Column(Object) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE dummy (\n\tpk STRING NOT NULL, \n\tobj_col OBJECT, ' @@ -92,7 +91,6 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -107,7 +105,6 @@ class DummyTable(self.Base): __tablename__ = 't' ts = sa.Column(sa.BigInteger, primary_key=True) p = sa.Column(sa.BigInteger, sa.Computed("date_trunc('day', ts)")) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -122,7 +119,6 @@ class DummyTable(self.Base): __tablename__ = 't' ts = sa.Column(sa.BigInteger, primary_key=True) p = sa.Column(sa.BigInteger, sa.Computed("date_trunc('day', ts)", persisted=False)) - with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) @@ -135,7 +131,6 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -171,7 +166,6 @@ class DummyTable(self.Base): } pk = sa.Column(sa.String, primary_key=True) p = sa.Column(sa.String, primary_key=True) - self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE t (\n\t' @@ -213,7 +207,6 @@ def test_column_pk_nullable(self): class DummyTable(self.Base): __tablename__ = 't' pk = sa.Column(sa.String, primary_key=True, nullable=True) - with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) @@ -237,7 +230,6 @@ class DummyTable(self.Base): __tablename__ = 't' pk = sa.Column(sa.String, primary_key=True) a = sa.Column(Geopoint, crate_index=False) - with self.assertRaises(sa.exc.CompileError): self.Base.metadata.create_all(bind=self.engine) From 29804b96dd0b47ae1bcc66a4c10f66d7d2872421 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 8 Jun 2023 20:10:22 +0200 Subject: [PATCH 5/5] Columnstore: Add changelog item --- CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 141b0667..0a7196f9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,9 @@ Changes for crate Unreleased ========== +- SQLAlchemy DDL: Allow turning off column store using ``crate_columnstore=False``. + Thanks, @fetzerms. + 2023/04/18 0.31.1 ================= pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy