Skip to content

Commit de8f2e9

Browse files
kwilcoxkwilcox
committed
Adds AVRO integration tests and improves the AVRO unit tests
1 parent ac8b4b2 commit de8f2e9

File tree

4 files changed

+138
-36
lines changed

4 files changed

+138
-36
lines changed

‎examples/integration_test.py

Lines changed: 90 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,12 @@
2020
""" Test script for confluent_kafka module """
2121

2222
import confluent_kafka
23-
import re
23+
import os
2424
import time
2525
import uuid
2626
import sys
2727
import json
28+
from copy import copy
2829

2930
try:
3031
from progress.bar import Bar
@@ -35,6 +36,9 @@
3536
# Kafka bootstrap server(s)
3637
bootstrap_servers = None
3738

39+
# Confluent schema-registry
40+
schema_registry = None
41+
3842
# Topic to use
3943
topic = 'test'
4044

@@ -50,7 +54,7 @@
5054
def error_cb (err):
5155
print('Error: %s' % err)
5256

53-
57+
5458
class MyTestDr(object):
5559
""" Producer: Delivery report callback """
5660

@@ -124,6 +128,83 @@ def verify_producer():
124128
p.flush()
125129

126130

131+
def verify_avro():
132+
from confluent_kafka import avro
133+
avsc_dir = os.path.join(os.path.dirname(__file__), os.pardir, 'tests', 'avro')
134+
135+
# Producer config
136+
conf = {'bootstrap.servers': bootstrap_servers,
137+
'schema.registry.url': schema_registry,
138+
'error_cb': error_cb,
139+
'api.version.request': api_version_request,
140+
'default.topic.config': {'produce.offset.report': True}}
141+
142+
# Create producer
143+
p = avro.AvroProducer(conf)
144+
145+
prim_float = avro.load(os.path.join(avsc_dir, "primitive_float.avsc"))
146+
prim_string = avro.load(os.path.join(avsc_dir, "primitive_string.avsc"))
147+
basic = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
148+
str_value = 'abc'
149+
float_value = 32.
150+
151+
combinations = [
152+
dict(key=float_value, key_schema=prim_float),
153+
dict(value=float_value, value_schema=prim_float),
154+
dict(key={'name': 'abc'}, key_schema=basic),
155+
dict(value={'name': 'abc'}, value_schema=basic),
156+
dict(value={'name': 'abc'}, value_schema=basic, key=float_value, key_schema=prim_float),
157+
dict(value={'name': 'abc'}, value_schema=basic, key=str_value, key_schema=prim_string),
158+
dict(value=float_value, value_schema=prim_float, key={'name': 'abc'}, key_schema=basic),
159+
dict(value=float_value, value_schema=prim_float, key=str_value, key_schema=prim_string),
160+
dict(value=str_value, value_schema=prim_string, key={'name': 'abc'}, key_schema=basic),
161+
dict(value=str_value, value_schema=prim_string, key=float_value, key_schema=prim_float),
162+
]
163+
164+
# Consumer config
165+
cons_conf = {'bootstrap.servers': bootstrap_servers,
166+
'schema.registry.url': schema_registry,
167+
'group.id': 'test.py',
168+
'session.timeout.ms': 6000,
169+
'enable.auto.commit': False,
170+
'api.version.request': api_version_request,
171+
'on_commit': print_commit_result,
172+
'error_cb': error_cb,
173+
'default.topic.config': {
174+
'auto.offset.reset': 'earliest'
175+
}}
176+
177+
for i, combo in enumerate(combinations):
178+
combo['topic'] = str(uuid.uuid4())
179+
p.produce(**combo)
180+
p.poll(0)
181+
p.flush()
182+
183+
# Create consumer
184+
c = avro.AvroConsumer(copy(cons_conf))
185+
c.subscribe([combo['topic']])
186+
187+
while True:
188+
msg = c.poll(0)
189+
if msg is None:
190+
continue
191+
192+
if msg.error():
193+
if msg.error().code() == confluent_kafka.KafkaError._PARTITION_EOF:
194+
break
195+
else:
196+
continue
197+
198+
tstype, timestamp = msg.timestamp()
199+
print('%s[%d]@%d: key=%s, value=%s, tstype=%d, timestamp=%s' %
200+
(msg.topic(), msg.partition(), msg.offset(),
201+
msg.key(), msg.value(), tstype, timestamp))
202+
203+
c.commit(msg, async=False)
204+
205+
# Close consumer
206+
c.close()
207+
127208

128209
def verify_producer_performance(with_dr_cb=True):
129210
""" Time how long it takes to produce and delivery X messages """
@@ -285,8 +366,6 @@ def verify_consumer():
285366
c.close()
286367

287368

288-
289-
290369
def verify_consumer_performance():
291370
""" Verify Consumer performance """
292371

@@ -450,8 +529,10 @@ def stats_cb(stats_json_str):
450529
bootstrap_servers = sys.argv[1]
451530
if len(sys.argv) > 2:
452531
topic = sys.argv[2]
532+
if len(sys.argv) > 3:
533+
schema_registry = sys.argv[3]
453534
else:
454-
print('Usage: %s <broker> [<topic>]' % sys.argv[0])
535+
print('Usage: %s <broker> [<topic>] [<schema_registry>]' % sys.argv[0])
455536
sys.exit(1)
456537

457538
print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version())
@@ -475,6 +556,8 @@ def stats_cb(stats_json_str):
475556
print('=' * 30, 'Verifying stats_cb', '=' * 30)
476557
verify_stats_cb()
477558

478-
print('=' * 30, 'Done', '=' * 30)
479-
559+
if schema_registry:
560+
print('=' * 30, 'Verifying AVRO', '=' * 30)
561+
topics = verify_avro()
480562

563+
print('=' * 30, 'Done', '=' * 30)

‎tests/avro/primitive_float.avsc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"type": "float"
3+
}

‎tests/avro/primitive_string.avsc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"type": "string"
3+
}

‎tests/avro/test_avro_producer.py

Lines changed: 42 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -24,53 +24,66 @@
2424

2525
from confluent_kafka import avro
2626

27+
from requests.exceptions import ConnectionError
2728
if sys.version_info[0] < 3:
2829
import unittest
2930
else:
3031
import unittest2 as unittest
31-
from confluent_kafka.avro import AvroProducer
3232

33+
from confluent_kafka.avro import AvroProducer
34+
from confluent_kafka.avro.serializer import (KeySerializerError,
35+
ValueSerializerError)
3336
avsc_dir = os.path.dirname(os.path.realpath(__file__))
3437

3538

3639
class TestAvroProducer(unittest.TestCase):
37-
def setUp(self):
38-
pass
3940

4041
def test_instantiation(self):
4142
obj = AvroProducer({'schema.registry.url': 'http://127.0.0.1:0'})
4243
self.assertTrue(isinstance(obj, AvroProducer))
4344
self.assertNotEqual(obj, None)
4445

45-
def test_Produce(self):
46-
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:0'})
47-
valueSchema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
48-
try:
49-
producer.produce(topic='test', value={"name": 'abc"'}, value_schema=valueSchema, key='mykey')
50-
self.fail("Should expect key_schema")
51-
except Exception as e:
52-
pass
53-
54-
def test_produce_arguments(self):
46+
def test_produce_no_key(self):
5547
value_schema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
56-
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:0'}, default_value_schema=value_schema)
48+
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:9001'}, default_value_schema=value_schema)
49+
with self.assertRaises(ConnectionError): # Unexistent schema-registry
50+
producer.produce(topic='test', value={"name": 'abc"'})
51+
52+
def test_produce_no_value(self):
53+
key_schema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
54+
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:9001'}, default_key_schema=key_schema)
55+
with self.assertRaises(ConnectionError): # Unexistent schema-registry
56+
producer.produce(topic='test', key={"name": 'abc"'})
5757

58-
try:
58+
def test_produce_no_value_schema(self):
59+
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:9001'})
60+
with self.assertRaises(ValueSerializerError):
61+
# Producer should not accept a value with no schema
5962
producer.produce(topic='test', value={"name": 'abc"'})
60-
except Exception as e:
61-
exc_type, exc_obj, exc_tb = sys.exc_info()
62-
if exc_type.__name__ == 'SerializerError':
63-
self.fail()
6463

65-
def test_produce_arguments_list(self):
66-
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:0'})
67-
try:
68-
producer.produce(topic='test', value={"name": 'abc"'}, key='mykey')
69-
except Exception as e:
70-
exc_type, exc_obj, exc_tb = sys.exc_info()
71-
if exc_type.__name__ == 'SerializerError':
72-
pass
64+
def test_produce_no_key_schema(self):
65+
value_schema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
66+
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:9001'}, default_value_schema=value_schema)
67+
with self.assertRaises(KeySerializerError):
68+
# If the key is provided as a dict an avro schema must also be provided
69+
producer.produce(topic='test', value={"name": 'abc"'}, key={"name": 'abc"'})
70+
71+
def test_produce_value_and_key_schemas(self):
72+
value_schema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
73+
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:9001'}, default_value_schema=value_schema, default_key_schema=value_schema)
74+
with self.assertRaises(ConnectionError): # Unexistent schema-registry
75+
producer.produce(topic='test', value={"name": 'abc"'}, key={"name": 'abc"'})
7376

77+
def test_produce_primitive_string_key(self):
78+
value_schema = avro.load(os.path.join(avsc_dir, "basic_schema.avsc"))
79+
key_schema = avro.load(os.path.join(avsc_dir, "primitive_string.avsc"))
80+
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:9001'})
81+
with self.assertRaises(ConnectionError): # Unexistent schema-registry
82+
producer.produce(topic='test', value={"name": 'abc"'}, value_schema=value_schema, key='mykey', key_schema=key_schema)
7483

75-
def suite():
76-
return unittest.TestLoader().loadTestsFromTestCase(TestAvroProducer)
84+
def test_produce_primitive_key_and_value(self):
85+
value_schema = avro.load(os.path.join(avsc_dir, "primitive_float.avsc"))
86+
key_schema = avro.load(os.path.join(avsc_dir, "primitive_string.avsc"))
87+
producer = AvroProducer({'schema.registry.url': 'http://127.0.0.1:9001'})
88+
with self.assertRaises(ConnectionError): # Unexistent schema-registry
89+
producer.produce(topic='test', value=32., value_schema=value_schema, key='mykey', key_schema=key_schema)

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy