From e435875da8954c1bd17eb3566ca1c113cd397aa4 Mon Sep 17 00:00:00 2001 From: Michaela Hojna Date: Mon, 10 Jan 2022 14:34:45 +0100 Subject: [PATCH 1/8] fix: ping function uses debug for log --- influxdb_client/client/influxdb_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/influxdb_client/client/influxdb_client.py b/influxdb_client/client/influxdb_client.py index 3acea596..e634c09f 100644 --- a/influxdb_client/client/influxdb_client.py +++ b/influxdb_client/client/influxdb_client.py @@ -429,7 +429,7 @@ def ping(self) -> bool: ping_service.get_ping() return True except Exception as ex: - logger.error("Unexpected error during /ping: %s", ex) + logger.debug("Unexpected error during /ping: %s", ex) return False def version(self) -> str: From c5b0756a8f36f6575f0c1f1e976462d35f6a75fc Mon Sep 17 00:00:00 2001 From: Michaela Hojna Date: Mon, 10 Jan 2022 14:57:57 +0100 Subject: [PATCH 2/8] docs: update CHANGELOG.md --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d835fd..fd050571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,10 @@ ### Bug Fixes 1. [#375](https://github.com/influxdata/influxdb-client-python/pull/375): Construct `InfluxDBError` without HTTP response -1. [#378](https://github.com/influxdata/influxdb-client-python/pull/378): Correct serialization DataFrame with nan values [DataFrame] -1. [#384](https://github.com/influxdata/influxdb-client-python/pull/384): Timeout can be specified as a `float` -1. [#380](https://github.com/influxdata/influxdb-client-python/pull/380): Correct data types for querying [DataFrame] +2. [#378](https://github.com/influxdata/influxdb-client-python/pull/378): Correct serialization DataFrame with nan values [DataFrame] +3. [#384](https://github.com/influxdata/influxdb-client-python/pull/384): Timeout can be specified as a `float` +4. [#380](https://github.com/influxdata/influxdb-client-python/pull/380): Correct data types for querying [DataFrame] +5. [#391](https://github.com/influxdata/influxdb-client-python/pull/391): Ping function uses debug for log ### CI 1. [#370](https://github.com/influxdata/influxdb-client-python/pull/370): Add Python 3.10 to CI builds From 7206969b4920662c4afbb08a890f3707b02c14cb Mon Sep 17 00:00:00 2001 From: Michaela Hojna Date: Mon, 10 Jan 2022 15:03:37 +0100 Subject: [PATCH 3/8] docs: update CHANGELOG.md --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd050571..b52c9324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,10 @@ ### Bug Fixes 1. [#375](https://github.com/influxdata/influxdb-client-python/pull/375): Construct `InfluxDBError` without HTTP response -2. [#378](https://github.com/influxdata/influxdb-client-python/pull/378): Correct serialization DataFrame with nan values [DataFrame] -3. [#384](https://github.com/influxdata/influxdb-client-python/pull/384): Timeout can be specified as a `float` -4. [#380](https://github.com/influxdata/influxdb-client-python/pull/380): Correct data types for querying [DataFrame] -5. [#391](https://github.com/influxdata/influxdb-client-python/pull/391): Ping function uses debug for log +1. [#378](https://github.com/influxdata/influxdb-client-python/pull/378): Correct serialization DataFrame with nan values [DataFrame] +1. [#384](https://github.com/influxdata/influxdb-client-python/pull/384): Timeout can be specified as a `float` +1. [#380](https://github.com/influxdata/influxdb-client-python/pull/380): Correct data types for querying [DataFrame] +1. [#391](https://github.com/influxdata/influxdb-client-python/pull/391): Ping function uses debug for log ### CI 1. [#370](https://github.com/influxdata/influxdb-client-python/pull/370): Add Python 3.10 to CI builds From cc264dddebb3484a4d8ac80901408bd1abf3ce16 Mon Sep 17 00:00:00 2001 From: Michaela Hojna Date: Thu, 13 Jan 2022 09:08:27 +0100 Subject: [PATCH 4/8] feat: added callback function for getting profilers --- examples/query_with_profilers.py | 42 +++++++++++++++++++++++ influxdb_client/client/flux_csv_parser.py | 31 +++++++++-------- influxdb_client/client/query_api.py | 11 +++--- tests/test_QueryApi.py | 21 ++++++++++++ 4 files changed, 87 insertions(+), 18 deletions(-) create mode 100644 examples/query_with_profilers.py diff --git a/examples/query_with_profilers.py b/examples/query_with_profilers.py new file mode 100644 index 00000000..5d0bbd24 --- /dev/null +++ b/examples/query_with_profilers.py @@ -0,0 +1,42 @@ +from influxdb_client import InfluxDBClient, Point +from influxdb_client.client.flux_table import FluxRecord +from influxdb_client.client.query_api import QueryOptions +from influxdb_client.client.write_api import SYNCHRONOUS + +with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=True) as client: + + """ + Define callback to process profiler results. + """ + class ProfilersCallback(object): + def __init__(self): + self.records = [] + + def __call__(self, flux_record): + self.records.append(flux_record.values) + + + callback = ProfilersCallback() + + write_api = client.write_api(write_options=SYNCHRONOUS) + + """ + Prepare data + """ + _point1 = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) + _point2 = Point("my_measurement").tag("location", "New York").field("temperature", 24.3) + write_api.write(bucket="my-bucket", record=[_point1, _point2]) + + """ + Pass callback to QueryOptions + """ + query_api = client.query_api( + query_options=QueryOptions(profilers=["query", "operator"], profiler_callback=callback)) + + """ + Perform query + """ + tables = query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') + + for profiler in callback.records: + print(f'Custom processing of profiler result: {profiler}') \ No newline at end of file diff --git a/influxdb_client/client/flux_csv_parser.py b/influxdb_client/client/flux_csv_parser.py index a973bef5..dbe3eb25 100644 --- a/influxdb_client/client/flux_csv_parser.py +++ b/influxdb_client/client/flux_csv_parser.py @@ -45,14 +45,15 @@ class FluxCsvParser(object): """Parse to processing response from InfluxDB to FluxStructures or DataFrame.""" def __init__(self, response: HTTPResponse, serialization_mode: FluxSerializationMode, - data_frame_index: List[str] = None, profilers: List[str] = None) -> None: + data_frame_index: List[str] = None, query_options: object = None) -> None: """Initialize defaults.""" self._response = response self.tables = [] self._serialization_mode = serialization_mode self._data_frame_index = data_frame_index self._data_frame_values = [] - self._profilers = profilers + self._profilers = query_options.profilers if query_options is not None else None + self._profiler_callback = query_options.profiler_callback if query_options is not None else None pass def __enter__(self): @@ -289,16 +290,18 @@ def table_list(self) -> List[FluxTable]: else: return list(filter(lambda table: not self._is_profiler_table(table), self.tables)) - @staticmethod - def _print_profiler_info(flux_record: FluxRecord): + def _print_profiler_info(self, flux_record: FluxRecord): if flux_record.get_measurement().startswith("profiler/"): - msg = "Profiler: " + flux_record.get_measurement() - print("\n" + len(msg) * "=") - print(msg) - print(len(msg) * "=") - for name in flux_record.values: - val = flux_record[name] - if isinstance(val, str) and len(val) > 50: - print(f"{name:<20}: \n\n{val}") - elif val is not None: - print(f"{name:<20}: {val:<20}") + if self._profiler_callback: + self._profiler_callback(flux_record) + else: + msg = "Profiler: " + flux_record.get_measurement() + print("\n" + len(msg) * "=") + print(msg) + print(len(msg) * "=") + for name in flux_record.values: + val = flux_record[name] + if isinstance(val, str) and len(val) > 50: + print(f"{name:<20}: \n\n{val}") + elif val is not None: + print(f"{name:<20}: {val:<20}") diff --git a/influxdb_client/client/query_api.py b/influxdb_client/client/query_api.py index 665651eb..b972d152 100644 --- a/influxdb_client/client/query_api.py +++ b/influxdb_client/client/query_api.py @@ -17,18 +17,21 @@ from influxdb_client.client.flux_table import FluxTable, FluxRecord from influxdb_client.client.util.date_utils import get_date_helper from influxdb_client.client.util.helpers import get_org_query_param +from typing import Callable class QueryOptions(object): """Query options.""" - def __init__(self, profilers: List[str] = None) -> None: + def __init__(self, profilers: List[str] = None, profiler_callback: Callable = None) -> None: """ Initialize query options. :param profilers: list of enabled flux profilers + :param profiler_callback: callback function return profilers (FluxRecord) """ self.profilers = profilers + self.profiler_callback = profiler_callback class QueryApi(object): @@ -101,7 +104,7 @@ def query(self, query: str, org=None, params: dict = None) -> List['FluxTable']: async_req=False, _preload_content=False, _return_http_data_only=False) _parser = FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.tables, - profilers=self._profilers()) + query_options=self._query_options) list(_parser.generator()) @@ -123,7 +126,7 @@ def query_stream(self, query: str, org=None, params: dict = None) -> Generator[' response = self._query_api.post_query(org=org, query=self._create_query(query, self.default_dialect, params), async_req=False, _preload_content=False, _return_http_data_only=False) _parser = FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.stream, - profilers=self._profilers()) + query_options=self._query_options) return _parser.generator() @@ -176,7 +179,7 @@ def query_data_frame_stream(self, query: str, org=None, data_frame_index: List[s _parser = FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.dataFrame, data_frame_index=data_frame_index, - profilers=self._profilers()) + query_options=self._query_options) return _parser.generator() def _profilers(self): diff --git a/tests/test_QueryApi.py b/tests/test_QueryApi.py index 29375203..bda2b068 100644 --- a/tests/test_QueryApi.py +++ b/tests/test_QueryApi.py @@ -376,6 +376,27 @@ def test_query_profiler_present(self): print(f"Profiler record: {flux_record}") self.assertTrue(found_profiler_records) + def test_profilers_callback(self): + + class ProfilersCallback(object): + def __init__(self): + self.records = [] + + def __call__(self, flux_record): + self.records.append(flux_record.values) + + def get_record(self, num, val): + return (self.records[num])[val] + + callback = ProfilersCallback() + + query_api = self.client.query_api(query_options=QueryOptions(profilers=["query", "operator"], + profiler_callback=callback)) + query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') + + self.assertEqual("profiler/query", callback.get_record(0, "_measurement")) + self.assertEqual("profiler/operator", callback.get_record(1, "_measurement")) + def test_profiler_ast(self): expect = { From f1006af3547d393426c8207ff12256d031746064 Mon Sep 17 00:00:00 2001 From: Michaela Hojna Date: Thu, 13 Jan 2022 10:05:27 +0100 Subject: [PATCH 5/8] docs: update CHANGELOG.md --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b52c9324..71e74e0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,13 @@ ### Bug Fixes 1. [#375](https://github.com/influxdata/influxdb-client-python/pull/375): Construct `InfluxDBError` without HTTP response -1. [#378](https://github.com/influxdata/influxdb-client-python/pull/378): Correct serialization DataFrame with nan values [DataFrame] -1. [#384](https://github.com/influxdata/influxdb-client-python/pull/384): Timeout can be specified as a `float` -1. [#380](https://github.com/influxdata/influxdb-client-python/pull/380): Correct data types for querying [DataFrame] -1. [#391](https://github.com/influxdata/influxdb-client-python/pull/391): Ping function uses debug for log +2. [#378](https://github.com/influxdata/influxdb-client-python/pull/378): Correct serialization DataFrame with nan values [DataFrame] +3. [#384](https://github.com/influxdata/influxdb-client-python/pull/384): Timeout can be specified as a `float` +4. [#380](https://github.com/influxdata/influxdb-client-python/pull/380): Correct data types for querying [DataFrame] +5. [#391](https://github.com/influxdata/influxdb-client-python/pull/391): Ping function uses debug for log + +### Features +1. [#393](https://github.com/influxdata/influxdb-client-python/pull/393): Added callback function for getting profilers: ### CI 1. [#370](https://github.com/influxdata/influxdb-client-python/pull/370): Add Python 3.10 to CI builds From 4bff0e8c18b182567cfab8eb1113b5d4038e2ee1 Mon Sep 17 00:00:00 2001 From: Michaela Hojna Date: Thu, 13 Jan 2022 10:54:45 +0100 Subject: [PATCH 6/8] fix: add new function for getting query options --- influxdb_client/client/query_api.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/influxdb_client/client/query_api.py b/influxdb_client/client/query_api.py index b972d152..8c41f208 100644 --- a/influxdb_client/client/query_api.py +++ b/influxdb_client/client/query_api.py @@ -104,7 +104,7 @@ def query(self, query: str, org=None, params: dict = None) -> List['FluxTable']: async_req=False, _preload_content=False, _return_http_data_only=False) _parser = FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.tables, - query_options=self._query_options) + query_options=self._get_query_options()) list(_parser.generator()) @@ -126,7 +126,7 @@ def query_stream(self, query: str, org=None, params: dict = None) -> Generator[' response = self._query_api.post_query(org=org, query=self._create_query(query, self.default_dialect, params), async_req=False, _preload_content=False, _return_http_data_only=False) _parser = FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.stream, - query_options=self._query_options) + query_options=self._get_query_options()) return _parser.generator() @@ -179,17 +179,18 @@ def query_data_frame_stream(self, query: str, org=None, data_frame_index: List[s _parser = FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.dataFrame, data_frame_index=data_frame_index, - query_options=self._query_options) + query_options=self._get_query_options()) return _parser.generator() - def _profilers(self): + def _get_query_options(self): if self._query_options and self._query_options.profilers: - return self._query_options.profilers - else: - return self._influxdb_client.profilers + return self._query_options + elif self._influxdb_client.profilers: + return QueryOptions(profilers=self._influxdb_client.profilers) def _create_query(self, query, dialect=default_dialect, params: dict = None): - profilers = self._profilers() + query_options = self._get_query_options() + profilers = query_options.profilers if query_options is not None else None q = Query(query=query, dialect=dialect, extern=QueryApi._build_flux_ast(params, profilers)) if profilers: From 830a70861f72055c7a2937f7d455957e0277f6e3 Mon Sep 17 00:00:00 2001 From: Michaela Hojna Date: Fri, 14 Jan 2022 09:04:30 +0100 Subject: [PATCH 7/8] fix: requested changes --- CHANGELOG.md | 8 +++--- README.rst | 30 +++++++++++++++++++++++ examples/README.md | 1 + examples/query_with_profilers.py | 3 +-- influxdb_client/client/flux_csv_parser.py | 2 +- influxdb_client/client/query_api.py | 3 +-- 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e01af1da..042e0b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,10 @@ ### Bug Fixes 1. [#375](https://github.com/influxdata/influxdb-client-python/pull/375): Construct `InfluxDBError` without HTTP response -2. [#378](https://github.com/influxdata/influxdb-client-python/pull/378): Correct serialization DataFrame with nan values [DataFrame] -3. [#384](https://github.com/influxdata/influxdb-client-python/pull/384): Timeout can be specified as a `float` -4. [#380](https://github.com/influxdata/influxdb-client-python/pull/380): Correct data types for querying [DataFrame] -5. [#391](https://github.com/influxdata/influxdb-client-python/pull/391): Ping function uses debug for log +1. [#378](https://github.com/influxdata/influxdb-client-python/pull/378): Correct serialization DataFrame with nan values [DataFrame] +1. [#384](https://github.com/influxdata/influxdb-client-python/pull/384): Timeout can be specified as a `float` +1. [#380](https://github.com/influxdata/influxdb-client-python/pull/380): Correct data types for querying [DataFrame] +1. [#391](https://github.com/influxdata/influxdb-client-python/pull/391): Ping function uses debug for log ### Features 1. [#393](https://github.com/influxdata/influxdb-client-python/pull/393): Added callback function for getting profilers with example and test diff --git a/README.rst b/README.rst index 76a0da0f..dd38eab6 100644 --- a/README.rst +++ b/README.rst @@ -337,6 +337,36 @@ Example of a profiler output: DurationSum : 940500 MeanDuration : 940500.0 +You can also use callback function to get profilers. +Return value of this callback is type of FluxRecord. + +Example how to use profilers with callback: + +.. code-block:: python + + class ProfilersCallback(object): + def __init__(self): + self.records = [] + + def __call__(self, flux_record): + self.records.append(flux_record.values) + + callback = ProfilersCallback() + + query_api = client.query_api(query_options=QueryOptions(profilers=["query", "operator"], profiler_callback=callback)) + tables = query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') + + for profiler in callback.records: + print(f'Custom processing of profiler result: {profiler}') + +Example output of this callback: + +.. code-block:: + + Custom processing of profiler result: {'result': '_profiler', 'table': 0, '_measurement': 'profiler/query', 'TotalDuration': 18843792, 'CompileDuration': 1078666, 'QueueDuration': 93375, 'PlanDuration': 0, 'RequeueDuration': 0, 'ExecuteDuration': 17371000, 'Concurrency': 0, 'MaxAllocated': 448, 'TotalAllocated': 0, 'RuntimeErrors': None, 'flux/query-plan': 'digraph {\r\n ReadRange2\r\n generated_yield\r\n\r\n ReadRange2 -> generated_yield\r\n}\r\n\r\n', 'influxdb/scanned-bytes': 0, 'influxdb/scanned-values': 0} + Custom processing of profiler result: {'result': '_profiler', 'table': 1, '_measurement': 'profiler/operator', 'Type': '*influxdb.readFilterSource', 'Label': 'ReadRange2', 'Count': 1, 'MinDuration': 3274084, 'MaxDuration': 3274084, 'DurationSum': 3274084, 'MeanDuration': 3274084.0} + + .. marker-index-end diff --git a/examples/README.md b/examples/README.md index 53b8370f..107e1e9f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -14,6 +14,7 @@ - [query.py](query.py) - How to query data into `FluxTable`s, `Stream` and `CSV` - [query_from_file.py](query_from_file.py) - How to use a Flux query defined in a separate file - [query_response_to_json.py](query_response_to_json.py) - How to serialize Query response to JSON +- [query_with_profilers.py](query_with_profilers.py) - How to get profilers records by callback ## Management API diff --git a/examples/query_with_profilers.py b/examples/query_with_profilers.py index 5d0bbd24..5275367b 100644 --- a/examples/query_with_profilers.py +++ b/examples/query_with_profilers.py @@ -1,5 +1,4 @@ from influxdb_client import InfluxDBClient, Point -from influxdb_client.client.flux_table import FluxRecord from influxdb_client.client.query_api import QueryOptions from influxdb_client.client.write_api import SYNCHRONOUS @@ -39,4 +38,4 @@ def __call__(self, flux_record): tables = query_api.query('from(bucket:"my-bucket") |> range(start: -10m)') for profiler in callback.records: - print(f'Custom processing of profiler result: {profiler}') \ No newline at end of file + print(f'Custom processing of profiler result: {profiler}') diff --git a/influxdb_client/client/flux_csv_parser.py b/influxdb_client/client/flux_csv_parser.py index dbe3eb25..67e08801 100644 --- a/influxdb_client/client/flux_csv_parser.py +++ b/influxdb_client/client/flux_csv_parser.py @@ -45,7 +45,7 @@ class FluxCsvParser(object): """Parse to processing response from InfluxDB to FluxStructures or DataFrame.""" def __init__(self, response: HTTPResponse, serialization_mode: FluxSerializationMode, - data_frame_index: List[str] = None, query_options: object = None) -> None: + data_frame_index: List[str] = None, query_options=None) -> None: """Initialize defaults.""" self._response = response self.tables = [] diff --git a/influxdb_client/client/query_api.py b/influxdb_client/client/query_api.py index 8c41f208..c4e65ce4 100644 --- a/influxdb_client/client/query_api.py +++ b/influxdb_client/client/query_api.py @@ -7,7 +7,7 @@ import codecs import csv from datetime import datetime, timedelta -from typing import List, Generator, Any, Union, Iterable +from typing import List, Generator, Any, Union, Iterable, Callable from influxdb_client import Dialect, IntegerLiteral, BooleanLiteral, FloatLiteral, DateTimeLiteral, StringLiteral, \ VariableAssignment, Identifier, OptionStatement, File, DurationLiteral, Duration, UnaryExpression, Expression, \ @@ -17,7 +17,6 @@ from influxdb_client.client.flux_table import FluxTable, FluxRecord from influxdb_client.client.util.date_utils import get_date_helper from influxdb_client.client.util.helpers import get_org_query_param -from typing import Callable class QueryOptions(object): From 7ba9349c4d9ea3e8aca35fd546af5341818ac083 Mon Sep 17 00:00:00 2001 From: Michaela Hojna Date: Fri, 14 Jan 2022 11:36:11 +0100 Subject: [PATCH 8/8] fix: requested changes --- CHANGELOG.md | 2 +- README.rst | 2 +- examples/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 042e0b26..152b3e6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ 1. [#391](https://github.com/influxdata/influxdb-client-python/pull/391): Ping function uses debug for log ### Features -1. [#393](https://github.com/influxdata/influxdb-client-python/pull/393): Added callback function for getting profilers with example and test +1. [#393](https://github.com/influxdata/influxdb-client-python/pull/393): Added callback function for getting profilers output with example and test ### CI 1. [#370](https://github.com/influxdata/influxdb-client-python/pull/370): Add Python 3.10 to CI builds diff --git a/README.rst b/README.rst index dd38eab6..738f99e5 100644 --- a/README.rst +++ b/README.rst @@ -337,7 +337,7 @@ Example of a profiler output: DurationSum : 940500 MeanDuration : 940500.0 -You can also use callback function to get profilers. +You can also use callback function to get profilers output. Return value of this callback is type of FluxRecord. Example how to use profilers with callback: diff --git a/examples/README.md b/examples/README.md index 107e1e9f..9892a837 100644 --- a/examples/README.md +++ b/examples/README.md @@ -14,7 +14,7 @@ - [query.py](query.py) - How to query data into `FluxTable`s, `Stream` and `CSV` - [query_from_file.py](query_from_file.py) - How to use a Flux query defined in a separate file - [query_response_to_json.py](query_response_to_json.py) - How to serialize Query response to JSON -- [query_with_profilers.py](query_with_profilers.py) - How to get profilers records by callback +- [query_with_profilers.py](query_with_profilers.py) - How to process profilers output by callback ## Management API pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy