Raise Syntax error for duplicate consumer/producer constructor parameters

rnpridgeon · rnpridgeon · commit 7ef6a369df10 · 2019-01-04T11:08:06.000-05:00
diff --git a/confluent_kafka/__init__.py b/confluent_kafka/__init__.py
@@ -1,4 +1,4 @@
-__all__ = ['cimpl', 'admin', 'avro', 'kafkatest']
+__all__ = ['cimpl', 'admin', 'avro', 'kafkatest', 'producer', 'consumer']
 from .cimpl import (KafkaError, # noqa
                     KafkaException,
                     Message,
diff --git a/confluent_kafka/consumer.py b/confluent_kafka/consumer.py
@@ -21,14 +21,6 @@
 from warnings import warn
 
 
-def byteDeserializer(topic, payload):
-    """
-        byteDeserializer returns an unaltered payload to the caller
-    """
-
-    return payload
-
-
 class Consumer(_impl):
     """
         Create a new Kafka Consumer instance.
@@ -53,15 +45,15 @@ class Consumer(_impl):
     """
     def __new__(cls, *args, **kwargs):
         if 'key_deserializer' in kwargs or 'value_deserializer' in kwargs:
-            return super(Consumer, cls).__new__(DeserializingConsumer, *args, **kwargs)
+            return super(Consumer, cls).__new__(SerializingConsumer, *args, **kwargs)
         return super(Consumer, cls).__new__(cls, *args, **kwargs)
 
 
-class DeserializingConsumer(Consumer):
+class SerializingConsumer(Consumer):
     """
-        DeserializingConsumer extends Consumer with configurable key and value deserializer.
+        SerializingConsumer extends Consumer with configurable key and value deserializer.
 
-        Instances of DeserializingConsumer cannot be created directly.
+        Instances of SerializingConsumer cannot be created directly.
         To obtain an instance of this class instantiate a Consumer with a key and/or value deserializer.
 
         Duplicate params have been omitted for brevity. See Consumer for class documentation.
@@ -73,10 +65,15 @@ class DeserializingConsumer(Consumer):
     __slots__ = ["_key_deserializer", "_value_deserializer"]
 
     def __new__(cls, *args, **kwargs):
-        raise TypeError("DeserializingConsumer is a non user-instantiable class")
+        raise TypeError("SerializingConsumer is a non user-instantiable class")
+
+    @staticmethod
+    def byteSerializer(topic, data):
+        """ Pass-through serializer """
+        return data
 
     # conf must remain optional as long as kwargs are supported
-    def __init__(self, conf={}, key_deserializer=byteDeserializer, value_deserializer=byteDeserializer,
+    def __init__(self, conf={}, key_deserializer=None, value_deserializer=None,
                  on_commit=None, stats_cb=None, throttle_cb=None, logger=None, **kwargs):
 
         if not isinstance(conf, dict):
@@ -90,26 +87,27 @@ def __init__(self, conf={}, key_deserializer=byteDeserializer, value_deserialize
                  "all keyword arguments must match the constructor signature explicitly.",
                  category=DeprecationWarning, stacklevel=2)
 
+        # Ensure the default serializer cannot be overwritten with None on instantiation
+        if key_deserializer is None:
+            key_deserializer = SerializingConsumer.byteSerializer
+
+        if value_deserializer is None:
+            value_deserializer = SerializingConsumer.byteSerializer
+
         self._key_deserializer = key_deserializer
         self._value_deserializer = value_deserializer
 
         # Callbacks can be set in the conf dict or *ideally* as parameters.
-        # Handle both cases prior to passing along to _impl
-        # If callbacks are configured in both places parameter values take precedence.
-        if not on_commit:
-            on_commit = conf.get('on_commit', None)
-
-        if not stats_cb:
-            stats_cb = conf.get('stats_cb', None)
-
-        if not throttle_cb:
-            throttle_cb = conf.get('throttle_cb', None)
-
-        if not logger:
-            logger = conf.get('logger', None)
+        # Raise a SyntaxError if a callback is set in both places.
+        for var, name in [(logger, 'logger'), (on_commit, 'on_commit'),
+                          (stats_cb, 'stats_cb'), (throttle_cb, 'throttle_cb')]:
+            if all([var, conf.get(name, None)]):
+                raise SyntaxError("{} parameter repeated".format(name))
+            if var is None:
+                var = conf.get(name, None)
 
-        super(DeserializingConsumer, self).__init__(conf, on_commit=on_commit, stats_cb=stats_cb,
-                                                    throttle_cb=throttle_cb, logger=logger)
+        super(SerializingConsumer, self).__init__(conf, on_commit=on_commit, stats_cb=stats_cb,
+                                                  throttle_cb=throttle_cb, logger=logger)
 
     def poll(self, timeout=-1.0):
         """
@@ -127,7 +125,7 @@ def poll(self, timeout=-1.0):
             :raises RuntimeError: If called on a closed consumer.
         """
 
-        msg = super(DeserializingConsumer, self).poll(timeout)
+        msg = super(SerializingConsumer, self).poll(timeout)
 
         if msg is None or msg.error():
             return msg
@@ -157,7 +155,7 @@ def consume(self, num_messages=1, timeout=-1):
             :raises ValueError: If num_messages > 1M.
         """
 
-        msgset = super(DeserializingConsumer, self).consume(num_messages, timeout)
+        msgset = super(SerializingConsumer, self).consume(num_messages, timeout)
         for msg in msgset:
             if msg.error():
                 continue
diff --git a/confluent_kafka/producer.py b/confluent_kafka/producer.py
@@ -21,14 +21,6 @@
 from warnings import warn
 
 
-def byteSerializer(topic, payload):
-    """
-        byteSerializer returns an unaltered payload to the caller
-    """
-
-    return payload
-
-
 class Producer(_impl):
     """
         Create a new Kafka Producer instance with or without serializer support.
@@ -78,8 +70,13 @@ class SerializingProducer(Producer):
     def __new__(cls, *args, **kwargs):
         raise TypeError("SerializingProducer is a non user-instantiable class")
 
+    @staticmethod
+    def byteSerializer(topic, data):
+        """ Pass-through serializer """
+        return data
+
     # conf must remain optional as long as kwargs are supported
-    def __init__(self, conf={}, key_serializer=byteSerializer, value_serializer=byteSerializer,
+    def __init__(self, conf={}, key_serializer=None, value_serializer=None,
                  error_cb=None, stats_cb=None, throttle_cb=None, logger=None, **kwargs):
 
         if not isinstance(conf, dict):
@@ -93,23 +90,24 @@ def __init__(self, conf={}, key_serializer=byteSerializer, value_serializer=byte
                  "all keyword arguments must match the constructor signature explicitly.",
                  category=DeprecationWarning, stacklevel=2)
 
+        # Ensure the default serializer cannot be overwritten with None on instantiation
+        if key_serializer is None:
+            key_serializer = SerializingProducer.byteSerializer
+
+        if value_serializer is None:
+            value_serializer = SerializingProducer.byteSerializer
+
         self._key_serializer = key_serializer
         self._value_serializer = value_serializer
 
         # Callbacks can be set in the conf dict or *ideally* as parameters.
-        # Handle both cases prior to passing along to _impl
-        # If callbacks are configured in both places parameter values take precedence.
-        if not error_cb:
-            error_cb = conf.get('error_cb', None)
-
-        if not stats_cb:
-            stats_cb = conf.get('stats_cb', None)
-
-        if not throttle_cb:
-            throttle_cb = conf.get('throttle_cb', None)
-
-        if not logger:
-            logger = conf.get('logger', None)
+        # Raise a SyntaxError if a callback is set in both places.
+        for var, name in [(logger, 'logger'), (error_cb, 'error_cb'),
+                          (stats_cb, 'stats_cb'), (throttle_cb, 'throttle_cb')]:
+            if all([var, conf.get(name, None)]):
+                raise SyntaxError("{} parameter repeated".format(name))
+            if var is None:
+                var = conf.get(name, None)
 
         super(SerializingProducer, self).__init__(conf, error_cb=error_cb, stats_cb=stats_cb,
                                                   throttle_cb=throttle_cb, logger=logger)
@@ -141,8 +139,5 @@ def produce(self, topic, value=None, key=None, partition=PARTITION_UA,
 
         """
 
-        key = self._key_serializer(topic, key)
-        value = self._value_serializer(topic, value)
-
-        super(SerializingProducer, self).produce(topic, value, key, partition, on_delivery=on_delivery,
+        super(SerializingProducer, self).produce(topic, self._value_serializer(topic, value), self._key_serializer(topic, key), partition, on_delivery=on_delivery,
                                                  callback=callback, timestamp=timestamp, headers=headers)

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-__all__ = ['cimpl', 'admin', 'avro', 'kafkatest']`
	`1`	`+__all__ = ['cimpl', 'admin', 'avro', 'kafkatest', 'producer', 'consumer']`
`2`	`2`	`from .cimpl import (KafkaError, # noqa`
`3`	`3`	`KafkaException,`
`4`	`4`	`Message,`