Skip to content

Commit c824d5a

Browse files
author
Peter Cornelius
committed
Resolve conflict
2 parents 5dd7bba + d395b73 commit c824d5a

27 files changed

+1453
-176
lines changed

‎.travis.yml

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,18 @@ matrix:
66
dist: trusty
77
python: "2.7"
88
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib" LIBRDKAFKA_VERSION=master
9-
# Source package verification with Python 2.7 and librdkafka v0.9.2
9+
# Source package verification with Python 2.7 and librdkafka v0.9.5
1010
- os: linux
1111
language: python
1212
dist: trusty
1313
python: "2.7"
14-
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib" LIBRDKAFKA_VERSION=v0.9.2
14+
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib" LIBRDKAFKA_VERSION=v0.9.5
1515
# Source package verification with Python 3.5 and librdkafka master
1616
- os: linux
1717
language: python
1818
dist: trusty
1919
python: "3.5"
20-
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib" LIBRDKAFKA_VERSION=v0.9.2
20+
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib" LIBRDKAFKA_VERSION=v0.9.5
2121
# cibuildwheel for osx
2222
- os: osx
2323
env: CIBW_BEFORE_BUILD="tools/bootstrap-librdkafka.sh --require-ssl master tmp" CFLAGS="-Itmp/include" LDFLAGS="-Ltmp/lib"
@@ -31,8 +31,10 @@ matrix:
3131
services: docker
3232

3333
install:
34+
- if [[ $TRAVIS_OS_NAME == "osx" ]]; then python -m ensurepip && virtualenv /tmp/venv && source /tmp/venv/bin/activate ; fi
35+
- pip install -U pip
36+
- if [[ -z $CIBW_BEFORE_BUILD ]]; then pip install pytest-timeout flake8 ; fi
3437
- if [[ -z $CIBW_BEFORE_BUILD ]]; then rm -rf tmp-build ; tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build ; fi
35-
- if [[ -z $CIBW_BEFORE_BUILD ]]; then pip install --upgrade pip && pip install pytest-timeout flake8 ; fi
3638
- if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install cibuildwheel==0.4.1 ; fi
3739

3840

@@ -47,9 +49,9 @@ script:
4749
deploy:
4850
provider: s3
4951
access_key_id:
50-
secure: TWG1fsfD+kWkTHB5RQNW9HDw6i7j7WlYs5oTPLLevOjFmuYhG4qCylp9ZYGqjMzL8Jl4MxydmNDSL/gXYDnNplOX9CA1qbp/pKIG1AjTda12JM6mxs9uZNUOrl4FYRteSfPP+N+25VhPA+ix8ffC/jZ/9wypdQMorn5aC6Ln0LlR3ZdG+JeiOdX2JeqO7d429iDyTzFWYZKJssW8GHGQj07OSVc3vQWBCUPJOaaVRsALKa3VZJgVXooXsjI2SORSnC4GAJhVgqzHLcs7AFVNaS2rrlF23mRcN8ePEnS76uIVk7Y2K90OeHcPgFfsTqrxFuLtHZYY68H2os+DamiERxD2xDpM28G2f3Ltb/bG7NGhuv0YHFWtwCbzB+aSJbGtUuFKqwdOwnD2PnQsVU0aX8N1lna3Fv4WVBBNXNeyKD0fbNYcTPvaOm+KSxN5/0l/BKyohjrTcVsn+6ZLdE6D5A0JfaX4sc3Lv2X+sdIc1ZLOL1OWg3eE1z4aAtKu3da8sPcP2Rb5V18uF0cBICdWP6kKtjRr+meCGw9S0Z2P3I1OLj+KfKmjrqtI7KXUTsz03npwOkbmFVprS5cXoW54kKhjrf99OhBXX7LhwJ5oKokBIDeXmUdyA+ettq5+5w5J8GgQ4MxIoY+vhy/wmcuM3yieASiftRPLFyJTr9/I6i8=
52+
secure: "T0hDlPMFjEliYnd9WjoKAvine5X7zLRLYzH5e2enzog747AveJJGU+I8Plgf+kfMj0hhin0tUWvuvfqEaRV5suSOLuO37Q+bmJRLHPf0ktpTZijXIVXGCw2X34LPyeOqv1XxbtJMD1p7FHEif9wHvUs3pwdBWE/iAwZTUr5B+pCTM02LrrskH1MA83rR/h98+ZK2G2yW+vJttRPIE4GDhG4ekOqkmho3fU2n7uCkEdnboGT/Kq5HkUcEyC9AjpnlLYzcJmgOPDNOKDMwb2pFjkRg1MfXcvaBoQT5bSSQyaS1WIqP9VfDi6oTl1vxuvg1AVLcXvZy9FzWFMjmghDxI/TLVAz0Tnh7RKL6WKqwFxyMRBJhxn81UkkG6c1kOlGDBABEySgAwKG01MkkwbPTaGU0mizxyvqj5JQNxZiS6vn0ZQHb3Y1YVeShdwkg2kb+G8MLKTAs9KYPQPdA5n91/g44BTTlMmy1JKNuBPfagrxyWdz9rDNG1l48HDnmSsERws8grJQh2kCCtON25GU7emtkjsBdfgZuZeVxKPdoW6CT5TUlh4Lo148Mm0OGIKgxsbQLJfRx3MGSBcCOYgYp0zdUDO5UKbli7Mv8A4kEgwXpGZGon20b5DD2O0e/DJWh9APnR/cfUN97OYqukmbk5SG9vt17MAtiVVDwj6UbrNE="
5153
secret_access_key:
52-
secure: G3KBqzmG7HaBznDMwNRq0qGIWiF3n96RgRnZzyvSfhRSqZq2Vv8llFo5GuCE0azzLA6Iot33x+Rugba3PbLReYoFOO87yWYT62Deq+lTwQHSEutcA1tNrW4ho+GPs2av/AhHE9e2r/xhp++2wHEvnj7Vp1HeVcmpAxkggZDkiS15GYiaVNEUa8tPL6GOcH3e8Xwcurr7dAi6KZnBFvylx0eADVPH3EAfN8TxapkT89FKjEtPZOV/srSFi5m561rW/P4+MwzBu0x01ouANdAKoAhEMubcEHANr1x8yuXlr99eapqnbfYMpaITyDbKIDSV/mVR5e/9G6xQEXs59FJJJ1bRCLF9uxIht3rlijED7WEElHJ1j2Ip9JZH0iKEPykw9vGNnjUZK2Hw4D8MvD+6jHbauZGsTeqTbh74tGxsIe/tpAZ2kmueqdO5iYK/R8louWjgmwGf+veqqQXCUw0UBnJoL5x+axp5nszQiepU/EnnyJ5bH1IB5qRUqesTYFWU/dxF7JQYie1wuAqn3ag40ir/Vg8stfjyNk8Ht0Ajpp+rgODzuURDiGutiblVpXffh83R5UZb21pI7oL6eE3HaeH20Ob6c8ELvWskM24EzS7baSJbrzatFtDw8hi0W3eeYIPqoeKC2btUavNnDkkP5mKPYQv0dsy754Yg3956SJw=
54+
secure: "imC8TwsarAxKhS7yMPNCwbvIhHmDhZB/NfWDLUToySP8ZQDWQMdlpiRlEA4RlNP8KOBGBShCX7dMXTC9nGFnbLhDKnpkZaQ3RIkXthjbcmsXiTAWNndVkCrgxDPpFwMHEs1Eu00AgJrELzDlKfhRls1txQO2N5wvVZFa1zyzlm46q4guesVakPVl3bZhx0gImYsJohXNVjvD3cks7+c6hFPDdMIEchgG+0amgCNNdK8ewZGfMMim5nvuHH00h8oJsIxQKOEzgz+JXwCi7hgd5UYn6DRFNyjmIK4iMphjC4pdKKp7r+m6Yj/s/Kc/YMVmRo3Zce4mM66KvmLSnnGGiewGRIkjyZ9UcvbrwkfudSMdPmWjnn0MKs7Vnfaej0xBZ9WN5c1QB6JzsLxStNp7gULfaZyqb7+zNtvYvbHYrFa80/RBpMbSBg6GSslsFmqEtsS9WInzTQYfNeSzrq5QU11ZKdsoQ53kruAtSxDJXFe1TKMSMnH/QWqvjwdrDoaJJZkJuMgNkznKuxorf7syF+mr9O63UKs72fHhEWBNOPfE+pwVlHUuBSE95jr5gu1Nxyyy5GV/sU+x/ik5a5Uytq9pcxhn4v3614VlG6t3Wy29BDDO54Sng0n12XXL2UaTYT6aIPTU1N8XQe66YjVvi7ms8Yt6HCA5+MMJdaSnkoI="
5355
bucket: librdkafka-ci-packages
5456
region: us-west-1
5557
local-dir: wheelhouse

‎README.md

Lines changed: 55 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,12 @@ Usage
3939
```python
4040
from confluent_kafka import Producer
4141

42+
4243
p = Producer({'bootstrap.servers': 'mybroker,mybroker2'})
44+
4345
for data in some_data_source:
4446
p.produce('mytopic', data.encode('utf-8'))
47+
4548
p.flush()
4649
```
4750

@@ -51,17 +54,29 @@ p.flush()
5154
```python
5255
from confluent_kafka import Consumer, KafkaError
5356

54-
c = Consumer({'bootstrap.servers': 'mybroker', 'group.id': 'mygroup',
55-
'default.topic.config': {'auto.offset.reset': 'smallest'}})
57+
58+
c = Consumer({
59+
'bootstrap.servers': 'mybroker',
60+
'group.id': 'mygroup',
61+
'default.topic.config': {
62+
'auto.offset.reset': 'smallest'
63+
}
64+
})
65+
5666
c.subscribe(['mytopic'])
57-
running = True
58-
while running:
67+
68+
while True:
5969
msg = c.poll()
60-
if not msg.error():
61-
print('Received message: %s' % msg.value().decode('utf-8'))
62-
elif msg.error().code() != KafkaError._PARTITION_EOF:
63-
print(msg.error())
64-
running = False
70+
71+
if msg.error():
72+
if msg.error().code() == KafkaError._PARTITION_EOF:
73+
continue
74+
else:
75+
print(msg.error())
76+
break
77+
78+
print('Received message: {}'.format(msg.value().decode('utf-8')))
79+
6580
c.close()
6681
```
6782

@@ -105,7 +120,11 @@ key_schema = avro.loads(key_schema_str)
105120
value = {"name": "Value"}
106121
key = {"name": "Key"}
107122

108-
avroProducer = AvroProducer({'bootstrap.servers': 'mybroker,mybroker2', 'schema.registry.url': 'http://schem_registry_host:port'}, default_key_schema=key_schema, default_value_schema=value_schema)
123+
avroProducer = AvroProducer({
124+
'bootstrap.servers': 'mybroker,mybroker2',
125+
'schema.registry.url': 'http://schem_registry_host:port'
126+
}, default_key_schema=key_schema, default_value_schema=value_schema)
127+
109128
avroProducer.produce(topic='my_topic', value=value, key=key)
110129
avroProducer.flush()
111130
```
@@ -117,21 +136,33 @@ from confluent_kafka import KafkaError
117136
from confluent_kafka.avro import AvroConsumer
118137
from confluent_kafka.avro.serializer import SerializerError
119138

120-
c = AvroConsumer({'bootstrap.servers': 'mybroker,mybroker2', 'group.id': 'groupid', 'schema.registry.url': 'http://127.0.0.1:8081'})
139+
140+
c = AvroConsumer({
141+
'bootstrap.servers': 'mybroker,mybroker2',
142+
'group.id': 'groupid',
143+
'schema.registry.url': 'http://127.0.0.1:8081'})
144+
121145
c.subscribe(['my_topic'])
122-
running = True
123-
while running:
146+
147+
while True:
124148
try:
125149
msg = c.poll(10)
126-
if msg:
127-
if not msg.error():
128-
print(msg.value())
129-
elif msg.error().code() != KafkaError._PARTITION_EOF:
130-
print(msg.error())
131-
running = False
150+
132151
except SerializerError as e:
133-
print("Message deserialization failed for %s: %s" % (msg, e))
134-
running = False
152+
print("Message deserialization failed for {}: {}".format(msg, e))
153+
break
154+
155+
if msg is None:
156+
continue
157+
158+
if msg.error():
159+
if msg.error().code() == KafkaError._PARTITION_EOF:
160+
continue
161+
else:
162+
print(msg.error())
163+
break
164+
165+
print(msg.value())
135166

136167
c.close()
137168
```
@@ -165,8 +196,8 @@ https://github.com/edenhill/librdkafka/wiki/Broker-version-compatibility
165196
Prerequisites
166197
=============
167198

168-
* Python >= 2.6 or Python 3.x
169-
* [librdkafka](https://github.com/edenhill/librdkafka) >= 0.9.1 (embedded in Linux wheels)
199+
* Python >= 2.7 or Python 3.x
200+
* [librdkafka](https://github.com/edenhill/librdkafka) >= 0.9.5 (latest release is embedded in wheels)
170201

171202
librdkafka is embedded in the manylinux wheels, for other platforms or
172203
when a specific version of librdkafka is desired, following these guidelines:
@@ -177,7 +208,7 @@ http://docs.confluent.io/current/installation.html#installation-apt
177208
* For **RedHat** and **RPM**-based distros, add this YUM repo and then do `sudo yum install librdkafka-devel python-devel`:
178209
http://docs.confluent.io/current/installation.html#rpm-packages-via-yum
179210

180-
* On **OSX**, use **homebrew** and do `sudo brew install librdkafka`
211+
* On **OSX**, use **homebrew** and do `brew install librdkafka`
181212

182213

183214
Install

‎confluent_kafka/avro/cached_schema_registry_client.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -168,9 +168,9 @@ def get_by_id(self, schema_id):
168168
# cache it
169169
self._cache_schema(result, schema_id)
170170
return result
171-
except:
171+
except ClientError as e:
172172
# bad schema - should not happen
173-
raise ClientError("Received bad schema from registry.")
173+
raise ClientError("Received bad schema (id %s) from registry: %s" % (schema_id, e))
174174

175175
def get_latest_schema(self, subject):
176176
"""
@@ -204,9 +204,9 @@ def get_latest_schema(self, subject):
204204
else:
205205
try:
206206
schema = loads(result['schema'])
207-
except:
207+
except ClientError:
208208
# bad schema - should not happen
209-
raise ClientError("Received bad schema from registry.")
209+
raise
210210

211211
self._cache_schema(schema, schema_id, subject, version)
212212
return (schema_id, schema, version)
@@ -269,7 +269,8 @@ def test_compatibility(self, subject, avro_schema, version='latest'):
269269
else:
270270
log.error("Unable to check the compatibility")
271271
False
272-
except:
272+
except Exception as e:
273+
log.error("_send_request() failed: %s", e)
273274
return False
274275

275276
def update_compatibility(self, level, subject=None):

‎confluent_kafka/avro/load.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,18 @@
1717

1818
import sys
1919

20+
from confluent_kafka.avro.error import ClientError
21+
2022

2123
def loads(schema_str):
2224
""" Parse a schema given a schema string """
23-
if sys.version_info[0] < 3:
24-
return schema.parse(schema_str)
25-
else:
26-
return schema.Parse(schema_str)
25+
try:
26+
if sys.version_info[0] < 3:
27+
return schema.parse(schema_str)
28+
else:
29+
return schema.Parse(schema_str)
30+
except schema.AvroException.SchemaParseException as e:
31+
raise ClientError("Schema parse failed: %s" % (str(e)))
2732

2833

2934
def load(fp):
@@ -44,5 +49,6 @@ def _hash_func(self):
4449
schema.RecordSchema.__hash__ = _hash_func
4550
schema.PrimitiveSchema.__hash__ = _hash_func
4651
schema.UnionSchema.__hash__ = _hash_func
52+
4753
except ImportError:
4854
schema = None

‎confluent_kafka/avro/serializer/message_serializer.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@
3939

4040
HAS_FAST = False
4141
try:
42-
from fastavro.reader import read_data
42+
from fastavro import schemaless_reader
4343

4444
HAS_FAST = True
45-
except:
45+
except ImportError:
4646
pass
4747

4848

@@ -158,29 +158,29 @@ def _get_decoder_func(self, schema_id, payload):
158158
# fetch from schema reg
159159
try:
160160
schema = self.registry_client.get_by_id(schema_id)
161-
except:
162-
schema = None
161+
except ClientError as e:
162+
raise SerializerError("unable to fetch schema with id %d: %s" % (schema_id, str(e)))
163163

164-
if not schema:
165-
err = "unable to fetch schema with id %d" % (schema_id)
166-
raise SerializerError(err)
164+
if schema is None:
165+
raise SerializerError("unable to fetch schema with id %d" % (schema_id))
167166

168167
curr_pos = payload.tell()
169168
if HAS_FAST:
170169
# try to use fast avro
171170
try:
172171
schema_dict = schema.to_json()
173-
read_data(payload, schema_dict)
172+
schemaless_reader(payload, schema_dict)
174173

175174
# If we reach this point, this means we have fastavro and it can
176175
# do this deserialization. Rewind since this method just determines
177176
# the reader function and we need to deserialize again along the
178177
# normal path.
179178
payload.seek(curr_pos)
180179

181-
self.id_to_decoder_func[schema_id] = lambda p: read_data(p, schema_dict)
180+
self.id_to_decoder_func[schema_id] = lambda p: schemaless_reader(p, schema_dict)
182181
return self.id_to_decoder_func[schema_id]
183-
except:
182+
except Exception:
183+
# Fast avro failed, fall thru to standard avro below.
184184
pass
185185

186186
# here means we should just delegate to slow avro

‎confluent_kafka/kafkatest/verifiable_client.py

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,17 +70,46 @@ def set_config(conf, args):
7070
for n, v in args.iteritems():
7171
if v is None:
7272
continue
73-
# Things to ignore
74-
if '.' not in n:
73+
74+
if not n.startswith('conf_'):
7575
# App config, skip
7676
continue
77-
if n.startswith('topic.'):
78-
# Set "topic.<...>" properties on default topic conf dict
79-
conf['default.topic.config'][n[6:]] = v
80-
elif n == 'partition.assignment.strategy':
77+
78+
# Remove conf_ prefix
79+
n = n[5:]
80+
81+
# Handle known Java properties to librdkafka properties.
82+
if n == 'partition.assignment.strategy':
8183
# Convert Java class name to config value.
8284
# "org.apache.kafka.clients.consumer.RangeAssignor" -> "range"
8385
conf[n] = re.sub(r'org.apache.kafka.clients.consumer.(\w+)Assignor',
8486
lambda x: x.group(1).lower(), v)
87+
88+
elif n == 'enable.idempotence':
89+
# Ignore idempotence for now, best-effortly.
90+
sys.stderr.write('%% WARN: Ignoring unsupported %s=%s\n' % (n, v))
8591
else:
8692
conf[n] = v
93+
94+
@staticmethod
95+
def read_config_file(path):
96+
"""Read (java client) config file and return dict with properties"""
97+
conf = {}
98+
99+
with open(path, 'r') as f:
100+
for line in f:
101+
line = line.strip()
102+
103+
if line.startswith('#') or len(line) == 0:
104+
continue
105+
106+
fi = line.find('=')
107+
if fi < 1:
108+
raise Exception('%s: invalid line, no key=value pair: %s' % (path, line))
109+
110+
k = line[:fi]
111+
v = line[fi+1:]
112+
113+
conf[k] = v
114+
115+
return conf

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy