Skip to content

Commit 454abee

Browse files
bednarbednar
authored
fix: Correctly parse CSV where multiple results include multiple tables (influxdata#64)
1 parent 27499f5 commit 454abee

File tree

3 files changed

+139
-3
lines changed

3 files changed

+139
-3
lines changed

‎CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
### Features
44
1. [#59](https://github.com/influxdata/influxdb-client-python/issues/59): Set User-Agent to influxdb-client-python/VERSION for all requests
55

6+
### Bugs
7+
1. [#61](https://github.com/influxdata/influxdb-client-python/issues/61): Correctly parse CSV where multiple results include multiple tables
8+
69
## 1.4.0 [2020-02-14]
710

811
### Features

‎influxdb_client/client/flux_csv_parser.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ def generator(self):
4949

5050
def _parse_flux_response(self):
5151
table_index = 0
52+
table_id = -1
5253
start_new_table = False
5354
table = None
5455
parsing_state_error = False
@@ -83,6 +84,7 @@ def _parse_flux_response(self):
8384
table = FluxTable()
8485
self._insert_table(table, table_index)
8586
table_index = table_index + 1
87+
table_id = -1
8688
elif table is None:
8789
raise FluxCsvParserException("Unable to parse CSV response. FluxTable definition was not found.")
8890

@@ -111,15 +113,18 @@ def _parse_flux_response(self):
111113
continue
112114

113115
# to int converions todo
114-
current_index = int(csv[2])
116+
current_id = int(csv[2])
117+
if table_id == -1:
118+
table_id = current_id
115119

116-
if current_index > (table_index - 1):
120+
if table_id != current_id:
117121
# create new table with previous column headers settings
118122
flux_columns = table.columns
119123
table = FluxTable()
120124
table.columns.extend(flux_columns)
121125
self._insert_table(table, table_index)
122126
table_index = table_index + 1
127+
table_id = current_id
123128

124129
flux_record = self.parse_record(table_index - 1, table, csv)
125130

@@ -217,4 +222,4 @@ def add_column_names_and_tags(table, csv):
217222

218223
def _insert_table(self, table, table_index):
219224
if self._serialization_mode is FluxSerializationMode.tables:
220-
self.tables.insert(table_index, table)
225+
self.tables.insert(table_index, table)

‎tests/test_FluxCSVParser.py

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
from io import BytesIO
2+
3+
from urllib3 import HTTPResponse
4+
5+
from influxdb_client.client.flux_csv_parser import FluxCsvParser, FluxSerializationMode
6+
from tests.base_test import BaseTest
7+
8+
9+
class FluxCsvParserTest(BaseTest):
10+
11+
def test_one_table(self):
12+
data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,long,long,string\n" \
13+
"#group,false,false,true,true,true,true,true,true,false,false,false\n" \
14+
"#default,_result,,,,,,,,,,\n" \
15+
",result,table,_start,_stop,_field,_measurement,host,region,_value2,value1,value_str\n" \
16+
",,0,1677-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,121,11,test\n"
17+
18+
tables = self._parse_to_tables(data=data)
19+
self.assertEqual(1, tables.__len__())
20+
self.assertEqual(11, tables[0].columns.__len__())
21+
self.assertEqual(1, tables[0].records.__len__())
22+
23+
def test_more_tables(self):
24+
data = "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,long,long,string\n" \
25+
"#group,false,false,true,true,true,true,true,true,false,false,false\n" \
26+
"#default,_result,,,,,,,,,,\n" \
27+
",result,table,_start,_stop,_field,_measurement,host,region,_value2,value1,value_str\n" \
28+
",,0,1677-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,A,west,121,11,test\n" \
29+
",,1,1677-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,free,mem,B,west,484,22,test\n" \
30+
",,2,1677-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,usage_system,cpu,A,west,1444,38,test\n" \
31+
",,3,1677-09-21T00:12:43.145224192Z,2018-07-16T11:21:02.547596934Z,user_usage,cpu,A,west,2401,49,test"
32+
33+
tables = self._parse_to_tables(data=data)
34+
self.assertEqual(4, tables.__len__())
35+
self.assertEqual(11, tables[0].columns.__len__())
36+
self.assertEqual(1, tables[0].records.__len__())
37+
self.assertEqual(11, tables[1].columns.__len__())
38+
self.assertEqual(1, tables[1].records.__len__())
39+
self.assertEqual(11, tables[2].columns.__len__())
40+
self.assertEqual(1, tables[2].records.__len__())
41+
self.assertEqual(11, tables[3].columns.__len__())
42+
self.assertEqual(1, tables[3].records.__len__())
43+
44+
def test_multiple_queries(self):
45+
data = "#datatype,string,long,string,string,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,double,string\n" \
46+
"#group,false,false,true,true,true,true,false,false,true\n" \
47+
"#default,t1,,,,,,,,\n" \
48+
",result,table,_field,_measurement,_start,_stop,_time,_value,tag\n" \
49+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:20:00Z,2,test1\n" \
50+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:21:40Z,2,test1\n" \
51+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:23:20Z,2,test1\n" \
52+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:25:00Z,2,test1\n" \
53+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:26:40Z,2,test1\n" \
54+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:28:20Z,2,test1\n" \
55+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:30:00Z,2,test1\n" \
56+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:20:00Z,2,test2\n" \
57+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:21:40Z,2,test2\n" \
58+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:23:20Z,2,test2\n" \
59+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:25:00Z,2,test2\n" \
60+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:26:40Z,2,test2\n" \
61+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:28:20Z,2,test2\n" \
62+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:30:00Z,2,test2\n" \
63+
"\n" \
64+
"#datatype,string,long,string,string,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,double,string\n" \
65+
"#group,false,false,true,true,true,true,false,false,true\n" \
66+
"#default,t2,,,,,,,,\n" \
67+
",result,table,_field,_measurement,_start,_stop,_time,_value,tag\n" \
68+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:20:00Z,2,test1\n" \
69+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:21:40Z,2,test1\n" \
70+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:23:20Z,2,test1\n" \
71+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:25:00Z,2,test1\n" \
72+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:26:40Z,2,test1\n" \
73+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:28:20Z,2,test1\n" \
74+
",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:30:00Z,2,test1\n" \
75+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:20:00Z,2,test2\n" \
76+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:21:40Z,2,test2\n" \
77+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:23:20Z,2,test2\n" \
78+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:25:00Z,2,test2\n" \
79+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:26:40Z,2,test2\n" \
80+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:28:20Z,2,test2\n" \
81+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:30:00Z,2,test2"
82+
83+
tables = self._parse_to_tables(data=data)
84+
self.assertEqual(4, tables.__len__())
85+
self.assertEqual(9, tables[0].columns.__len__())
86+
self.assertEqual(7, tables[0].records.__len__())
87+
self.assertEqual(9, tables[1].columns.__len__())
88+
self.assertEqual(7, tables[1].records.__len__())
89+
self.assertEqual(9, tables[2].columns.__len__())
90+
self.assertEqual(7, tables[2].records.__len__())
91+
self.assertEqual(9, tables[3].columns.__len__())
92+
self.assertEqual(7, tables[3].records.__len__())
93+
94+
def test_table_index_not_start_at_zero(self):
95+
data = "#datatype,string,long,string,string,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,double,string\n" \
96+
"#group,false,false,true,true,true,true,false,false,true\n" \
97+
"#default,t1,,,,,,,,\n" \
98+
",result,table,_field,_measurement,_start,_stop,_time,_value,tag\n" \
99+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:20:00Z,2,test1\n" \
100+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:21:40Z,2,test1\n" \
101+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:23:20Z,2,test1\n" \
102+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:25:00Z,2,test1\n" \
103+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:26:40Z,2,test1\n" \
104+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:28:20Z,2,test1\n" \
105+
",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:30:00Z,2,test1\n" \
106+
",,2,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:20:00Z,2,test2\n" \
107+
",,2,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:21:40Z,2,test2\n" \
108+
",,2,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:23:20Z,2,test2\n" \
109+
",,2,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:25:00Z,2,test2\n" \
110+
",,2,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:26:40Z,2,test2\n" \
111+
",,2,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:28:20Z,2,test2\n" \
112+
",,2,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:30:00Z,2,test2\n"
113+
114+
tables = self._parse_to_tables(data=data)
115+
self.assertEqual(2, tables.__len__())
116+
self.assertEqual(9, tables[0].columns.__len__())
117+
self.assertEqual(7, tables[0].records.__len__())
118+
self.assertEqual(9, tables[1].columns.__len__())
119+
self.assertEqual(7, tables[1].records.__len__())
120+
121+
@staticmethod
122+
def _parse_to_tables(data: str):
123+
fp = BytesIO(str.encode(data))
124+
_parser = FluxCsvParser(response=HTTPResponse(fp, preload_content=False),
125+
serialization_mode=FluxSerializationMode.tables)
126+
list(_parser.generator())
127+
tables = _parser.tables
128+
return tables

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy