From c6334f8cbb13586d0aa53135f340e0d3edf162c8 Mon Sep 17 00:00:00 2001 From: "tzonghao.chen" Date: Fri, 9 Sep 2016 11:54:14 -0400 Subject: [PATCH 1/2] Fix DataFrameClient empty tag processing Empty or None tags should be left out, otherwise it would break line protocol. --- influxdb/_dataframe_client.py | 14 ++++++++++--- influxdb/tests/dataframe_client_test.py | 27 ++++++++++++++++--------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 0341a41d..9eec9254 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -279,13 +279,21 @@ def _convert_dataframe_to_lines(self, # If tag columns exist, make an array of formatted tag keys and values if tag_columns: tag_df = dataframe[tag_columns] + tag_df = tag_df.fillna('') # replace NA with empty string tag_df = tag_df.sort_index(axis=1) tag_df = self._stringify_dataframe( tag_df, numeric_precision, datatype='tag') - tags = (',' + ( - (tag_df.columns.values + '=').tolist() + tag_df)).sum(axis=1) - del tag_df + # prepend tag keys + tag_df = tag_df.apply( + lambda s: s.apply( + lambda v, l: l + '=' + v if v else None, l=s.name)) + + # join tags, but leave out None values + tags = tag_df.apply( + lambda r: ',' + ','.join(r.dropna()), axis=1) + + del tag_df else: tags = '' diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index 782e5c82..eb2e292c 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -201,20 +201,29 @@ def test_write_points_from_dataframe_with_tag_cols_and_defaults(self): def test_write_points_from_dataframe_with_tag_escaped(self): now = pd.Timestamp('1970-01-01 00:00+00:00') dataframe = pd.DataFrame( - data=[['blue', 1, "1", 1, 1.0, 'hot'], - ['red,green=orange', 0, "2", 2, 2.0, 'cold']], - index=[now, now + timedelta(hours=1)], - columns=["tag_one", "tag_two", "column_one", - "column_two", "column_three", - "tag_three"]) + data=[ + ['blue orange', "1", 1, 'hot=cold'], # space, equal + ['red,green', "2", 2, r'cold\fire'], # comma, backslash + ['some', "2", 2, ''], # skip empty + ['some', "2", 2, None], # skip None + ], + index=pd.period_range(now, freq='H', periods=4), + columns=["tag_one", "column_one", "column_two", "tag_three"] + ) expected_escaped_tags = ( - b"foo,tag_one=blue " + b"foo,tag_one=blue\\ orange,tag_three=hot\\=cold " b"column_one=\"1\",column_two=1i " b"0\n" - b"foo,tag_one=red\\,green\\=orange " + b"foo,tag_one=red\\,green,tag_three=cold\\\\fire " b"column_one=\"2\",column_two=2i " b"3600000000000\n" + b"foo,tag_one=some " + b"column_one=\"2\",column_two=2i " + b"7200000000000\n" + b"foo,tag_one=some " + b"column_one=\"2\",column_two=2i " + b"10800000000000\n" ) with requests_mock.Mocker() as m: @@ -224,7 +233,7 @@ def test_write_points_from_dataframe_with_tag_escaped(self): cli = DataFrameClient(database='db') cli.write_points(dataframe, 'foo', field_columns=['column_one', 'column_two'], - tag_columns=['tag_one']) + tag_columns=['tag_one', 'tag_three']) self.assertEqual(m.last_request.body, expected_escaped_tags) def test_write_points_from_dataframe_with_numeric_column_names(self): From a7c64405af3e819f6d4b864d5eb7b38a1395bea0 Mon Sep 17 00:00:00 2001 From: "tzonghao.chen" Date: Fri, 9 Sep 2016 12:12:08 -0400 Subject: [PATCH 2/2] Handle all-empty tags --- influxdb/_dataframe_client.py | 5 ++++- influxdb/tests/dataframe_client_test.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py index 9eec9254..f854f82c 100644 --- a/influxdb/_dataframe_client.py +++ b/influxdb/_dataframe_client.py @@ -291,7 +291,10 @@ def _convert_dataframe_to_lines(self, # join tags, but leave out None values tags = tag_df.apply( - lambda r: ',' + ','.join(r.dropna()), axis=1) + lambda r: ','.join(r.dropna()), axis=1) + + # prepend comma + tags = tags.apply(lambda v: ',' + v if v else '') del tag_df else: diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py index eb2e292c..3667b0fc 100644 --- a/influxdb/tests/dataframe_client_test.py +++ b/influxdb/tests/dataframe_client_test.py @@ -206,8 +206,9 @@ def test_write_points_from_dataframe_with_tag_escaped(self): ['red,green', "2", 2, r'cold\fire'], # comma, backslash ['some', "2", 2, ''], # skip empty ['some', "2", 2, None], # skip None + ['', "2", 2, None], # all tags empty ], - index=pd.period_range(now, freq='H', periods=4), + index=pd.period_range(now, freq='H', periods=5), columns=["tag_one", "column_one", "column_two", "tag_three"] ) @@ -224,6 +225,9 @@ def test_write_points_from_dataframe_with_tag_escaped(self): b"foo,tag_one=some " b"column_one=\"2\",column_two=2i " b"10800000000000\n" + b"foo " + b"column_one=\"2\",column_two=2i " + b"14400000000000\n" ) with requests_mock.Mocker() as m: pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy