Skip to content

Commit 4f275a2

Browse files
author
aviau
committed
Merge branch 'cannium-InfluxDBClusterClient' into HEAD
2 parents f3d6143 + 7632c33 commit 4f275a2

File tree

4 files changed

+294
-18
lines changed

4 files changed

+294
-18
lines changed

‎README.rst

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,20 @@ Here's a basic example (for more see the examples directory)::
126126

127127
>>> print("Result: {0}".format(result))
128128

129+
If you want to connect to a cluster, you could initialize a ``InfluxDBClusterClient``::
130+
131+
$ python
132+
133+
>>> from influxdb import InfluxDBClusterClient
134+
135+
>>> cc = InfluxDBClusterClient(hosts = [('192.168.0.1', 8086),
136+
('192.168.0.2', 8086),
137+
('192.168.0.3', 8086)],
138+
username='root',
139+
password='root',
140+
database='example')
141+
142+
``InfluxDBClusterClient`` has the same methods as ``InfluxDBClient``, it basically is a proxy to multiple InfluxDBClients.
129143

130144
Testing
131145
=======

‎influxdb/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
# -*- coding: utf-8 -*-
22
from .client import InfluxDBClient
3+
from .client import InfluxDBClusterClient
34
from .dataframe_client import DataFrameClient
45
from .helper import SeriesHelper
56

67

78
__all__ = [
89
'InfluxDBClient',
10+
'InfluxDBClusterClient',
911
'DataFrameClient',
1012
'SeriesHelper',
1113
]

‎influxdb/client.py

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
Python client for InfluxDB
44
"""
55
from collections import OrderedDict
6+
from functools import wraps
67
import json
78
import socket
9+
import random
810
import requests
911
import requests.exceptions
1012
from sys import version_info
@@ -33,6 +35,12 @@ def __init__(self, content, code):
3335
self.code = code
3436

3537

38+
class InfluxDBServerError(Exception):
39+
"""Raised when server error occurs"""
40+
def __init__(self, content):
41+
super(InfluxDBServerError, self).__init__(content)
42+
43+
3644
class InfluxDBClient(object):
3745

3846
"""
@@ -485,3 +493,114 @@ def send_packet(self, packet):
485493
data = json.dumps(packet)
486494
byte = data.encode('utf-8')
487495
self.udp_socket.sendto(byte, (self._host, self.udp_port))
496+
497+
498+
class InfluxDBClusterClient(object):
499+
"""
500+
The ``InfluxDBClusterClient`` is the client for connecting to a cluster of
501+
InfluxDB Servers. It basically is a proxy to multiple ``InfluxDBClient``s.
502+
503+
:param hosts: A list of hosts, where a host should be in format
504+
(address, port)
505+
e.g. [('127.0.0.1', 8086), ('127.0.0.1', 9096)]
506+
:param shuffle: If true, queries will hit servers evenly(randomly)
507+
:param client_base_class: In order to support different clients,
508+
default to InfluxDBClient
509+
"""
510+
511+
def __init__(self,
512+
hosts=[('localhost', 8086)],
513+
username='root',
514+
password='root',
515+
database=None,
516+
ssl=False,
517+
verify_ssl=False,
518+
timeout=None,
519+
use_udp=False,
520+
udp_port=4444,
521+
shuffle=True,
522+
client_base_class=InfluxDBClient,
523+
):
524+
self.clients = []
525+
self.bad_clients = [] # Corresponding server has failures in history
526+
self.shuffle = shuffle
527+
for h in hosts:
528+
self.clients.append(client_base_class(host=h[0], port=h[1],
529+
username=username,
530+
password=password,
531+
database=database,
532+
ssl=ssl,
533+
verify_ssl=verify_ssl,
534+
timeout=timeout,
535+
use_udp=use_udp,
536+
udp_port=udp_port))
537+
for method in dir(client_base_class):
538+
if method.startswith('_'):
539+
continue
540+
orig_func = getattr(client_base_class, method)
541+
if not callable(orig_func):
542+
continue
543+
setattr(self, method, self._make_func(orig_func))
544+
545+
@staticmethod
546+
def from_DSN(dsn, client_base_class=InfluxDBClient,
547+
shuffle=True, **kwargs):
548+
"""
549+
Same as InfluxDBClient.from_DSN, and supports multiple servers.
550+
551+
Example DSN:
552+
influxdb://usr:pwd@host1:8086,usr:pwd@host2:8086/db_name
553+
udp+influxdb://usr:pwd@host1:8086,usr:pwd@host2:8086/db_name
554+
https+influxdb://usr:pwd@host1:8086,usr:pwd@host2:8086/db_name
555+
556+
:param shuffle: If true, queries will hit servers evenly(randomly)
557+
:param client_base_class: In order to support different clients,
558+
default to InfluxDBClient
559+
"""
560+
dsn = dsn.lower()
561+
conn_params = urlparse(dsn)
562+
netlocs = conn_params.netloc.split(',')
563+
cluster_client = InfluxDBClusterClient(
564+
hosts=[],
565+
client_base_class=client_base_class,
566+
shuffle=shuffle,
567+
**kwargs)
568+
for netloc in netlocs:
569+
single_dsn = '%(scheme)s://%(netloc)s%(path)s' % (
570+
{'scheme': conn_params.scheme,
571+
'netloc': netloc,
572+
'path': conn_params.path}
573+
)
574+
cluster_client.clients.append(client_base_class.from_DSN(
575+
single_dsn,
576+
**kwargs))
577+
return cluster_client
578+
579+
def _make_func(self, orig_func):
580+
581+
@wraps(orig_func)
582+
def func(*args, **kwargs):
583+
if self.shuffle:
584+
random.shuffle(self.clients)
585+
clients = self.clients + self.bad_clients
586+
for c in clients:
587+
bad_client = False
588+
try:
589+
return orig_func(c, *args, **kwargs)
590+
except InfluxDBClientError as e:
591+
# Errors caused by user's requests, re-raise
592+
raise e
593+
except Exception as e:
594+
# Errors that might caused by server failure, try another
595+
bad_client = True
596+
if c in self.clients:
597+
self.clients.remove(c)
598+
self.bad_clients.append(c)
599+
finally:
600+
if not bad_client and c in self.bad_clients:
601+
self.bad_clients.remove(c)
602+
self.clients.append(c)
603+
604+
raise InfluxDBServerError("InfluxDB: no viable server!")
605+
606+
return func

‎tests/influxdb/client_test.py

Lines changed: 159 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@
2626
import warnings
2727
import mock
2828

29-
from influxdb import InfluxDBClient
29+
from influxdb import InfluxDBClient, InfluxDBClusterClient
30+
from influxdb.client import InfluxDBServerError
3031

3132

3233
def _build_response_object(status_code=200, content=""):
@@ -96,41 +97,41 @@ def setUp(self):
9697

9798
def test_scheme(self):
9899
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
99-
assert cli._baseurl == 'http://host:8086'
100+
self.assertEqual('http://host:8086', cli._baseurl)
100101

101102
cli = InfluxDBClient(
102103
'host', 8086, 'username', 'password', 'database', ssl=True
103104
)
104-
assert cli._baseurl == 'https://host:8086'
105+
self.assertEqual('https://host:8086', cli._baseurl)
105106

106107
def test_dsn(self):
107108
cli = InfluxDBClient.from_DSN('influxdb://usr:pwd@host:1886/db')
108-
assert cli._baseurl == 'http://host:1886'
109-
assert cli._username == 'usr'
110-
assert cli._password == 'pwd'
111-
assert cli._database == 'db'
112-
assert cli.use_udp is False
109+
self.assertEqual('http://host:1886', cli._baseurl)
110+
self.assertEqual('usr', cli._username)
111+
self.assertEqual('pwd', cli._password)
112+
self.assertEqual('db', cli._database)
113+
self.assertFalse(cli.use_udp)
113114

114115
cli = InfluxDBClient.from_DSN('udp+influxdb://usr:pwd@host:1886/db')
115-
assert cli.use_udp is True
116+
self.assertTrue(cli.use_udp)
116117

117118
cli = InfluxDBClient.from_DSN('https+influxdb://usr:pwd@host:1886/db')
118-
assert cli._baseurl == 'https://host:1886'
119+
self.assertEqual('https://host:1886', cli._baseurl)
119120

120121
cli = InfluxDBClient.from_DSN('https+influxdb://usr:pwd@host:1886/db',
121122
**{'ssl': False})
122-
assert cli._baseurl == 'http://host:1886'
123+
self.assertEqual('http://host:1886', cli._baseurl)
123124

124125
def test_switch_database(self):
125126
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
126127
cli.switch_database('another_database')
127-
assert cli._database == 'another_database'
128+
self.assertEqual('another_database', cli._database)
128129

129130
def test_switch_user(self):
130131
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
131132
cli.switch_user('another_username', 'another_password')
132-
assert cli._username == 'another_username'
133-
assert cli._password == 'another_password'
133+
self.assertEqual('another_username', cli._username)
134+
self.assertEqual('another_password', cli._password)
134135

135136
def test_write(self):
136137
with requests_mock.Mocker() as m:
@@ -207,10 +208,8 @@ def test_write_points_toplevel_attributes(self):
207208
def test_write_points_batch(self):
208209
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
209210
with _mocked_session(cli, 'post', 200, self.dummy_points):
210-
assert cli.write_points(
211-
data=self.dummy_points,
212-
batch_size=2
213-
) is True
211+
self.assertTrue(cli.write_points(data=self.dummy_points,
212+
batch_size=2))
214213

215214
def test_write_points_udp(self):
216215
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
@@ -565,3 +564,145 @@ def test_get_list_users_empty(self):
565564
)
566565

567566
self.assertListEqual(self.cli.get_list_users(), [])
567+
568+
569+
class FakeClient(InfluxDBClient):
570+
fail = False
571+
572+
def query(self,
573+
query,
574+
params={},
575+
expected_response_code=200,
576+
database=None):
577+
if query == 'Fail':
578+
raise Exception("Fail")
579+
580+
if self.fail:
581+
raise Exception("Fail")
582+
else:
583+
return "Success"
584+
585+
586+
class TestInfluxDBClusterClient(unittest.TestCase):
587+
588+
def setUp(self):
589+
# By default, raise exceptions on warnings
590+
warnings.simplefilter('error', FutureWarning)
591+
592+
self.hosts = [('host1', 8086), ('host2', 8086), ('host3', 8086)]
593+
594+
def test_init(self):
595+
cluster = InfluxDBClusterClient(hosts=self.hosts,
596+
username='username',
597+
password='password',
598+
database='database',
599+
shuffle=False,
600+
client_base_class=FakeClient)
601+
self.assertEqual(3, len(cluster.clients))
602+
self.assertEqual(0, len(cluster.bad_clients))
603+
for idx, client in enumerate(cluster.clients):
604+
self.assertEqual(self.hosts[idx][0], client._host)
605+
self.assertEqual(self.hosts[idx][1], client._port)
606+
607+
def test_one_server_fails(self):
608+
cluster = InfluxDBClusterClient(hosts=self.hosts,
609+
database='database',
610+
shuffle=False,
611+
client_base_class=FakeClient)
612+
cluster.clients[0].fail = True
613+
self.assertEqual('Success', cluster.query(''))
614+
self.assertEqual(2, len(cluster.clients))
615+
self.assertEqual(1, len(cluster.bad_clients))
616+
617+
def test_two_servers_fail(self):
618+
cluster = InfluxDBClusterClient(hosts=self.hosts,
619+
database='database',
620+
shuffle=False,
621+
client_base_class=FakeClient)
622+
cluster.clients[0].fail = True
623+
cluster.clients[1].fail = True
624+
self.assertEqual('Success', cluster.query(''))
625+
self.assertEqual(1, len(cluster.clients))
626+
self.assertEqual(2, len(cluster.bad_clients))
627+
628+
def test_all_fail(self):
629+
cluster = InfluxDBClusterClient(hosts=self.hosts,
630+
database='database',
631+
shuffle=True,
632+
client_base_class=FakeClient)
633+
with self.assertRaises(InfluxDBServerError):
634+
cluster.query('Fail')
635+
self.assertEqual(0, len(cluster.clients))
636+
self.assertEqual(3, len(cluster.bad_clients))
637+
638+
def test_all_good(self):
639+
cluster = InfluxDBClusterClient(hosts=self.hosts,
640+
database='database',
641+
shuffle=True,
642+
client_base_class=FakeClient)
643+
self.assertEqual('Success', cluster.query(''))
644+
self.assertEqual(3, len(cluster.clients))
645+
self.assertEqual(0, len(cluster.bad_clients))
646+
647+
def test_recovery(self):
648+
cluster = InfluxDBClusterClient(hosts=self.hosts,
649+
database='database',
650+
shuffle=True,
651+
client_base_class=FakeClient)
652+
with self.assertRaises(InfluxDBServerError):
653+
cluster.query('Fail')
654+
self.assertEqual('Success', cluster.query(''))
655+
self.assertEqual(1, len(cluster.clients))
656+
self.assertEqual(2, len(cluster.bad_clients))
657+
658+
def test_dsn(self):
659+
cli = InfluxDBClusterClient.from_DSN(
660+
'influxdb://usr:pwd@host1:8086,usr:pwd@host2:8086/db')
661+
self.assertEqual(2, len(cli.clients))
662+
self.assertEqual('http://host1:8086', cli.clients[0]._baseurl)
663+
self.assertEqual('usr', cli.clients[0]._username)
664+
self.assertEqual('pwd', cli.clients[0]._password)
665+
self.assertEqual('db', cli.clients[0]._database)
666+
self.assertFalse(cli.clients[0].use_udp)
667+
self.assertEqual('http://host2:8086', cli.clients[1]._baseurl)
668+
self.assertEqual('usr', cli.clients[1]._username)
669+
self.assertEqual('pwd', cli.clients[1]._password)
670+
self.assertEqual('db', cli.clients[1]._database)
671+
self.assertFalse(cli.clients[1].use_udp)
672+
673+
cli = InfluxDBClusterClient.from_DSN(
674+
'udp+influxdb://usr:pwd@host1:8086,usr:pwd@host2:8086/db')
675+
self.assertTrue(cli.clients[0].use_udp)
676+
self.assertTrue(cli.clients[1].use_udp)
677+
678+
cli = InfluxDBClusterClient.from_DSN(
679+
'https+influxdb://usr:pwd@host1:8086,usr:pwd@host2:8086/db')
680+
self.assertEqual('https://host1:8086', cli.clients[0]._baseurl)
681+
self.assertEqual('https://host2:8086', cli.clients[1]._baseurl)
682+
683+
cli = InfluxDBClusterClient.from_DSN(
684+
'https+influxdb://usr:pwd@host1:8086,usr:pwd@host2:8086/db',
685+
**{'ssl': False})
686+
self.assertEqual('http://host1:8086', cli.clients[0]._baseurl)
687+
self.assertEqual('http://host2:8086', cli.clients[1]._baseurl)
688+
689+
def test_dsn_single_client(self):
690+
cli = InfluxDBClusterClient.from_DSN('influxdb://usr:pwd@host:8086/db')
691+
self.assertEqual('http://host:8086', cli.clients[0]._baseurl)
692+
self.assertEqual('usr', cli.clients[0]._username)
693+
self.assertEqual('pwd', cli.clients[0]._password)
694+
self.assertEqual('db', cli.clients[0]._database)
695+
self.assertFalse(cli.clients[0].use_udp)
696+
697+
cli = InfluxDBClusterClient.from_DSN(
698+
'udp+influxdb://usr:pwd@host:8086/db')
699+
self.assertTrue(cli.clients[0].use_udp)
700+
701+
cli = InfluxDBClusterClient.from_DSN(
702+
'https+influxdb://usr:pwd@host:8086/db')
703+
self.assertEqual('https://host:8086', cli.clients[0]._baseurl)
704+
705+
cli = InfluxDBClusterClient.from_DSN(
706+
'https+influxdb://usr:pwd@host:8086/db',
707+
**{'ssl': False})
708+
self.assertEqual('http://host:8086', cli.clients[0]._baseurl)

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy