Line protocol leading comma (influxdata#694)

d3banjan · aviau · commit 08e02996a17c · 2019-07-11T18:33:24.000-04:00
* [fix] typo in comment + [fix] handles leading comma for the case that the first value column is Null valued * [refactor] consolidated similar logic to a new function * [fix] covering scenario where is a string * Revert "[fix] covering scenario where is a string" This reverts commit 49af5ab. * Revert "[refactor] consolidated similar logic to a new function" This reverts commit 366e771. * [tests][feature] added tests to check if first none value results in invalid line protocol * [fix] deleted debug lines * [fix] overspecified date_range args * [fix] overspecified date_range args * [fix] removed endpoint in date-range * [fix] reordered columns in test target * [fix] [test] freeze order of columns * [refactor] [test] used loc instead of dict-like invocation of columns * [fix] [test] [lint] cleared up complainsts from flake8 and pep257
diff --git a/influxdb/_dataframe_client.py b/influxdb/_dataframe_client.py
@@ -363,7 +363,7 @@ def _convert_dataframe_to_lines(self,
             tag_df = self._stringify_dataframe(
                 tag_df, numeric_precision, datatype='tag')
 
-            # join preprendded tags, leaving None values out
+            # join prepended tags, leaving None values out
             tags = tag_df.apply(
                 lambda s: [',' + s.name + '=' + v if v else '' for v in s])
             tags = tags.sum(axis=1)
@@ -392,6 +392,8 @@ def _convert_dataframe_to_lines(self,
             field_df.columns[1:]]
         field_df = field_df.where(~mask_null, '')  # drop Null entries
         fields = field_df.sum(axis=1)
+        # take out leading , where first column has a Null value
+        fields = fields.str.lstrip(",")
         del field_df
 
         # Generate line protocol string
diff --git a/influxdb/tests/dataframe_client_test.py b/influxdb/tests/dataframe_client_test.py
@@ -389,6 +389,71 @@ def test_write_points_from_dataframe_with_numeric_column_names(self):
 
             self.assertEqual(m.last_request.body, expected)
 
+    def test_write_points_from_dataframe_with_leading_none_column(self):
+        """write_points detect erroneous leading comma for null first field."""
+        dataframe = pd.DataFrame(
+            dict(
+                first=[1, None, None, 8, 9],
+                second=[2, None, None, None, 10],
+                third=[3, 4.1, None, None, 11],
+                first_tag=["one", None, None, "eight", None],
+                second_tag=["two", None, None, None, None],
+                third_tag=["three", "four", None, None, None],
+                comment=[
+                    "All columns filled",
+                    "First two of three empty",
+                    "All empty",
+                    "Last two of three empty",
+                    "Empty tags with values",
+                ]
+            ),
+            index=pd.date_range(
+                start=pd.to_datetime('2018-01-01'),
+                freq='1D',
+                periods=5,
+            )
+        )
+        expected = (
+            b'foo,first_tag=one,second_tag=two,third_tag=three'
+            b' comment="All columns filled",first=1.0,second=2.0,third=3.0'
+            b' 1514764800000000000\n'
+            b'foo,third_tag=four'
+            b' comment="First two of three empty",third=4.1'
+            b' 1514851200000000000\n'
+            b'foo comment="All empty" 1514937600000000000\n'
+            b'foo,first_tag=eight'
+            b' comment="Last two of three empty",first=8.0'
+            b' 1515024000000000000\n'
+            b'foo'
+            b' comment="Empty tags with values",first=9.0,second=10.0'
+            b',third=11.0'
+            b' 1515110400000000000\n'
+        )
+
+        with requests_mock.Mocker() as m:
+            m.register_uri(requests_mock.POST,
+                           "http://localhost:8086/write",
+                           status_code=204)
+
+            cli = DataFrameClient(database='db')
+
+            colnames = [
+                "first_tag",
+                "second_tag",
+                "third_tag",
+                "comment",
+                "first",
+                "second",
+                "third"
+            ]
+            cli.write_points(dataframe.loc[:, colnames], 'foo',
+                             tag_columns=[
+                                 "first_tag",
+                                 "second_tag",
+                                 "third_tag"])
+
+            self.assertEqual(m.last_request.body, expected)
+
     def test_write_points_from_dataframe_with_numeric_precision(self):
         """Test write points from df with numeric precision."""
         now = pd.Timestamp('1970-01-01 00:00+00:00')
