Skip to content

Commit 1f3f205

Browse files
rnpridgeonRyan P
rnpridgeon
authored and
Ryan P
committed
Complete HTTPS support for schema registry (confluentinc#90)
update tox
1 parent 1937de2 commit 1f3f205

File tree

7 files changed

+250
-44
lines changed

7 files changed

+250
-44
lines changed

‎confluent_kafka/avro/__init__.py

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,20 @@ class AvroProducer(Producer):
2828

2929
def __init__(self, config, default_key_schema=None,
3030
default_value_schema=None, schema_registry=None):
31+
3132
schema_registry_url = config.pop("schema.registry.url", None)
33+
schema_registry_ca_location = config.pop("schema.registry.ssl.ca.location", None)
34+
schema_registry_certificate_location = config.pop("schema.registry.ssl.certificate.location", None)
35+
schema_registry_key_location = config.pop("schema.registry.ssl.key.location", None)
36+
3237
if schema_registry is None:
3338
if schema_registry_url is None:
3439
raise ValueError("Missing parameter: schema.registry.url")
35-
schema_registry = CachedSchemaRegistryClient(url=schema_registry_url)
40+
41+
schema_registry = CachedSchemaRegistryClient(url=schema_registry_url,
42+
ca_location=schema_registry_ca_location,
43+
cert_location=schema_registry_certificate_location,
44+
key_location=schema_registry_key_location)
3645
elif schema_registry_url is not None:
3746
raise ValueError("Cannot pass schema_registry along with schema.registry.url config")
3847

@@ -92,11 +101,20 @@ class AvroConsumer(Consumer):
92101
and the standard Kafka client configuration (``bootstrap.servers`` et.al).
93102
"""
94103
def __init__(self, config, schema_registry=None):
104+
95105
schema_registry_url = config.pop("schema.registry.url", None)
106+
schema_registry_ca_location = config.pop("schema.registry.ssl.ca.location", None)
107+
schema_registry_certificate_location = config.pop("schema.registry.ssl.certificate.location", None)
108+
schema_registry_key_location = config.pop("schema.registry.ssl.key.location", None)
109+
96110
if schema_registry is None:
97111
if schema_registry_url is None:
98112
raise ValueError("Missing parameter: schema.registry.url")
99-
schema_registry = CachedSchemaRegistryClient(url=schema_registry_url)
113+
114+
schema_registry = CachedSchemaRegistryClient(url=schema_registry_url,
115+
ca_location=schema_registry_ca_location,
116+
cert_location=schema_registry_certificate_location,
117+
key_location=schema_registry_key_location)
100118
elif schema_registry_url is not None:
101119
raise ValueError("Cannot pass schema_registry along with schema.registry.url config")
102120

‎confluent_kafka/avro/cached_schema_registry_client.py

Lines changed: 33 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
from . import loads
3030

3131
VALID_LEVELS = ['NONE', 'FULL', 'FORWARD', 'BACKWARD']
32+
VALID_METHODS = ['GET', 'POST', 'PUT', 'DELETE']
3233

3334
# Common accept header sent
3435
ACCEPT_HDR = "application/vnd.schemaregistry.v1+json, application/vnd.schemaregistry+json, application/json"
@@ -46,7 +47,7 @@ class CachedSchemaRegistryClient(object):
4647
@:param: url: url to schema registry
4748
"""
4849

49-
def __init__(self, url, max_schemas_per_subject=1000):
50+
def __init__(self, url, max_schemas_per_subject=1000, ca_location=None, cert_location=None, key_location=None):
5051
"""Construct a client by passing in the base URL of the schema registry server"""
5152

5253
self.url = url.rstrip('/')
@@ -59,32 +60,41 @@ def __init__(self, url, max_schemas_per_subject=1000):
5960
# subj => { schema => version }
6061
self.subject_to_schema_versions = defaultdict(dict)
6162

62-
def _send_request(self, url, method='GET', body=None, headers=None):
63-
if body:
64-
body = json.dumps(body)
65-
body = body.encode('utf8')
66-
_headers = dict()
67-
_headers["Accept"] = ACCEPT_HDR
63+
s = requests.Session()
64+
if ca_location is not None:
65+
s.verify = ca_location
66+
if cert_location is not None or key_location is not None:
67+
if cert_location is None or key_location is None:
68+
raise ValueError(
69+
"Both schema.registry.ssl.certificate.location and schema.registry.ssl.key.location must be set")
70+
s.cert = (cert_location, key_location)
71+
72+
self._session = s
73+
74+
def __del__(self):
75+
self.close()
76+
77+
def __enter__(self):
78+
return self
79+
80+
def __exit__(self, *args):
81+
self.close()
82+
83+
def close(self):
84+
self._session.close()
85+
86+
def _send_request(self, url, method='GET', body=None, headers={}):
87+
if method not in VALID_METHODS:
88+
raise ClientError("Method {} is invalid; valid methods include {}".format(method, VALID_METHODS))
89+
90+
_headers = {'Accept': ACCEPT_HDR}
6891
if body:
6992
_headers["Content-Length"] = str(len(body))
7093
_headers["Content-Type"] = "application/vnd.schemaregistry.v1+json"
94+
_headers.update(headers)
7195

72-
if headers:
73-
for header_name in headers:
74-
_headers[header_name] = headers[header_name]
75-
if method == 'GET':
76-
response = requests.get(url, headers=_headers)
77-
elif method == 'POST':
78-
response = requests.post(url, body, headers=_headers)
79-
elif method == 'PUT':
80-
response = requests.put(url, body, headers=_headers)
81-
elif method == 'DELETE':
82-
response = requests.delete(url, headers=_headers)
83-
else:
84-
raise ClientError("Invalid HTTP request type")
85-
86-
result = json.loads(response.text)
87-
return (result, response.status_code)
96+
response = self._session.request(method, url, headers=_headers, json=body)
97+
return response.json(), response.status_code
8898

8999
def _add_to_cache(self, cache, subject, schema, value):
90100
sub_cache = cache[subject]

‎examples/integration_test.py

Lines changed: 130 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,17 @@
4545
except ImportError as e:
4646
with_progress = False
4747

48+
# Default test conf location
49+
testconf = "tests/testconf.json"
50+
4851
# Kafka bootstrap server(s)
4952
bootstrap_servers = None
5053

5154
# Confluent schema-registry
5255
schema_registry_url = None
5356

5457
# Topic prefix to use
55-
topic = 'test'
58+
topic = None
5659

5760
# API version requests are only implemented in Kafka broker >=0.10
5861
# but the client handles failed API version requests gracefully for older
@@ -374,6 +377,96 @@ def verify_avro():
374377
c.close()
375378

376379

380+
def verify_avro_https():
381+
from confluent_kafka import avro
382+
avsc_dir = os.path.join(os.path.dirname(__file__), os.pardir, 'tests', 'avro')
383+
384+
# Producer config
385+
conf = {'bootstrap.servers': bootstrap_servers,
386+
'error_cb': error_cb,
387+
'api.version.request': api_version_request}
388+
389+
conf.update(testconf.get('schema_registry_https', {}))
390+
391+
p = avro.AvroProducer(conf)
392+
393+
prim_float = avro.load(os.path.join(avsc_dir, "primitive_float.avsc"))
394+
prim_string = avro.load(os.path.join(avsc_dir, "primitive_string.avsc"))
395+
basic = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
396+
str_value = 'abc'
397+
float_value = 32.0
398+
399+
combinations = [
400+
dict(key=float_value, key_schema=prim_float),
401+
dict(value=float_value, value_schema=prim_float),
402+
dict(key={'name': 'abc'}, key_schema=basic),
403+
dict(value={'name': 'abc'}, value_schema=basic),
404+
dict(value={'name': 'abc'}, value_schema=basic, key=float_value, key_schema=prim_float),
405+
dict(value={'name': 'abc'}, value_schema=basic, key=str_value, key_schema=prim_string),
406+
dict(value=float_value, value_schema=prim_float, key={'name': 'abc'}, key_schema=basic),
407+
dict(value=float_value, value_schema=prim_float, key=str_value, key_schema=prim_string),
408+
dict(value=str_value, value_schema=prim_string, key={'name': 'abc'}, key_schema=basic),
409+
dict(value=str_value, value_schema=prim_string, key=float_value, key_schema=prim_float),
410+
# Verify identity check allows Falsy object values(e.g., 0, empty string) to be handled properly (issue #342)
411+
dict(value='', value_schema=prim_string, key=0.0, key_schema=prim_float),
412+
dict(value=0.0, value_schema=prim_float, key='', key_schema=prim_string),
413+
]
414+
415+
for i, combo in enumerate(combinations):
416+
combo['topic'] = str(uuid.uuid4())
417+
combo['headers'] = [('index', str(i))]
418+
p.produce(**combo)
419+
p.flush()
420+
421+
conf = {'bootstrap.servers': bootstrap_servers,
422+
'group.id': generate_group_id(),
423+
'session.timeout.ms': 6000,
424+
'enable.auto.commit': False,
425+
'api.version.request': api_version_request,
426+
'on_commit': print_commit_result,
427+
'error_cb': error_cb,
428+
'default.topic.config': {
429+
'auto.offset.reset': 'earliest'
430+
}}
431+
432+
conf.update(testconf.get('schema_registry_https', {}))
433+
434+
c = avro.AvroConsumer(conf)
435+
c.subscribe([(t['topic']) for t in combinations])
436+
437+
msgcount = 0
438+
while msgcount < len(combinations):
439+
msg = c.poll(0)
440+
441+
if msg is None or msg.error():
442+
continue
443+
444+
tstype, timestamp = msg.timestamp()
445+
print('%s[%d]@%d: key=%s, value=%s, tstype=%d, timestamp=%s' %
446+
(msg.topic(), msg.partition(), msg.offset(),
447+
msg.key(), msg.value(), tstype, timestamp))
448+
449+
# omit empty Avro fields from payload for comparison
450+
record_key = msg.key()
451+
record_value = msg.value()
452+
index = int(dict(msg.headers())['index'])
453+
454+
if isinstance(msg.key(), dict):
455+
record_key = {k: v for k, v in msg.key().items() if v is not None}
456+
457+
if isinstance(msg.value(), dict):
458+
record_value = {k: v for k, v in msg.value().items() if v is not None}
459+
460+
assert combinations[index].get('key') == record_key
461+
assert combinations[index].get('value') == record_value
462+
463+
c.commit()
464+
msgcount += 1
465+
466+
# Close consumer
467+
c.close()
468+
469+
377470
def verify_producer_performance(with_dr_cb=True):
378471
""" Time how long it takes to produce and delivery X messages """
379472
conf = {'bootstrap.servers': bootstrap_servers,
@@ -1125,7 +1218,7 @@ def verify_config(expconfig, configs):
11251218

11261219
# Exclude throttle since from default list
11271220
default_modes = ['consumer', 'producer', 'avro', 'performance', 'admin']
1128-
all_modes = default_modes + ['throttle', 'none']
1221+
all_modes = default_modes + ['throttle', 'avro-https', 'none']
11291222
"""All test modes"""
11301223

11311224

@@ -1140,6 +1233,21 @@ def print_usage(exitcode, reason=None):
11401233
sys.exit(exitcode)
11411234

11421235

1236+
def generate_group_id():
1237+
return str(uuid.uuid1())
1238+
1239+
1240+
def resolve_envs(_conf):
1241+
"""Resolve environment variables"""
1242+
1243+
for k, v in _conf.items():
1244+
if isinstance(v, dict):
1245+
resolve_envs(v)
1246+
1247+
if str(v).startswith('$'):
1248+
_conf[k] = os.getenv(v[1:])
1249+
1250+
11431251
if __name__ == '__main__':
11441252
"""Run test suites"""
11451253

@@ -1152,22 +1260,30 @@ def print_usage(exitcode, reason=None):
11521260
# Parse options
11531261
while len(sys.argv) > 1 and sys.argv[1].startswith('--'):
11541262
opt = sys.argv.pop(1)[2:]
1263+
1264+
if opt == 'conf':
1265+
testconf = sys.argv.pop(1)
1266+
continue
1267+
11551268
if opt not in all_modes:
11561269
print_usage(1, 'unknown option --' + opt)
11571270
modes.append(opt)
11581271

1159-
if len(sys.argv) > 1:
1160-
bootstrap_servers = sys.argv[1]
1161-
if len(sys.argv) > 2:
1162-
topic = sys.argv[2]
1163-
if len(sys.argv) > 3:
1164-
schema_registry_url = sys.argv[3]
1165-
else:
1166-
print_usage(1)
1272+
with open(testconf) as f:
1273+
testconf = json.load(f)
1274+
resolve_envs(testconf)
1275+
1276+
bootstrap_servers = testconf.get('bootstrap.servers', None)
1277+
topic = testconf.get('topic', None)
1278+
schema_registry_url = testconf.get('schema.registry.url', None)
11671279

11681280
if len(modes) == 0:
11691281
modes = default_modes
11701282

1283+
if bootstrap_servers is None or topic is None:
1284+
print_usage(1, "Properties bootstrap.servers and topic must be set. "
1285+
"Use tests/testconf-example.json as a template when creating a new conf file.")
1286+
11711287
print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version())
11721288
print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion())
11731289
print('Testing: %s' % modes)
@@ -1213,6 +1329,10 @@ def print_usage(exitcode, reason=None):
12131329
print('=' * 30, 'Verifying AVRO', '=' * 30)
12141330
verify_avro()
12151331

1332+
if 'avro-https' in modes:
1333+
print('=' * 30, 'Verifying AVRO with HTTPS', '=' * 30)
1334+
verify_avro_https()
1335+
12161336
if 'admin' in modes:
12171337
print('=' * 30, 'Verifying Admin API', '=' * 30)
12181338
verify_admin()

‎tests/README.md

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,30 +10,36 @@ From top-level directory run:
1010
Integration tests
1111
=================
1212

13-
**WARNING**: These tests require an active Kafka cluster and will create new topics.
13+
**NOTE**: Integration tests require an existing Kafka cluster and a `testconf.json` configuration file. Any value provided
14+
in `testconf.json` prefixed with '$' will be treated as an environment variable and automatically resolved.
1415

16+
At a minimum you must specify `bootstrap.servers` and `topic` within `testconf.json`. Please reference [tests/testconf-example.json](tests/testconf-example.json) for formatting.
1517

16-
To run all of the integration test `modes` uncomment the following line from `tox.ini` and add the addresses to your Kafka and Confluent Schema Registry instances.
18+
**WARNING**: These tests will create new topics and consumer groups.
1719

18-
#python examples/integration_test.py <bootstrap-servers> confluent-kafka-testing [<schema-registry-url>]
20+
To run all of the integration test `modes` uncomment the following line from `tox.ini` and provide the location to your `testconf.json`
21+
22+
#python examples/integration_test.py --conf <testconf.json>
1923

2024
You can also run the integration tests outside of `tox` by running this command from the source root directory
2125

22-
examples/integration_test.py <kafka-broker> [<test-topic>] [<schema-registry>]
26+
python examples/integration_test.py --conf <testconf.json>
2327

2428
To run individual integration test `modes` use the following syntax
2529

26-
examples/integration_test.py --<test mode> <kafka-broker> [<test-topic>] [<schema-registry>]
30+
python examples/integration_test.py --<test mode> --conf <testconf.json>
2731

2832
For example:
2933

30-
examples/integration_test.py --producer <kafka-broker> [<test-topic>]
34+
python examples/integration_test.py --producer --conf testconf.json
3135

3236
To get a list of modes you can run the integration test manually with the `--help` flag
3337

34-
examples/integration_tests.py --help
38+
python examples/integration_tests.py --help
3539

3640

41+
Throttle Callback test
42+
======================
3743
The throttle_cb integration test requires an additional step and as such is not included in the default test modes.
3844
In order to execute the throttle_cb test you must first set a throttle for the client 'throttled_client' with the command below:
3945

@@ -43,11 +49,26 @@ In order to execute the throttle_cb test you must first set a throttle for the c
4349

4450
Once the throttle has been set you can proceed with the following command:
4551

46-
examples/integration_test.py --throttle <kafka-broker> [<test-topic>]
52+
python examples/integration_test.py --throttle --conf testconf.json
4753

4854

4955
To remove the throttle you can execute the following
5056

5157
kafka-configs --zookeeper <zookeeper host>:<zookeeper port> \
5258
--alter --delete-config 'request_percentage' \
5359
--entity-name throttled_client --entity-type clients
60+
61+
62+
HTTPS Schema Registry test
63+
==========================
64+
65+
HTTPS tests require access to a Schema Registry instance configured to with at least one HTTPS listener.
66+
67+
For instructions on how to configure the Schema Registry please see the Confluent documentation:
68+
69+
[Schema Registry documentation](https://docs.confluent.io/current/schema-registry/docs/security.html#configuring-the-rest-api-for-http-or-https)
70+
71+
If client authentication has been enabled you will need to provide both the client certificate, `schema.registry.ssl.certificate.location`,
72+
and the client's private key, `schema.registry.ssl.key.location`
73+
74+
python examples/integration_test.py --avro-https --conf testconf.json

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy