Merge pull request confluentinc#140 from confluentinc/drerr84

edenhill · web-flow · commit 4859e7feaa54 · 2017-02-28T22:43:29.000+01:00
Handle delivery.report.only.error in Python (confluentinc#84)
diff --git a/confluent_kafka/src/Producer.c b/confluent_kafka/src/Producer.c
@@ -152,6 +152,10 @@ static void dr_msg_cb (rd_kafka_t *rk, const rd_kafka_message_t *rkm,
 		goto done;
 	}
 
+        /* Skip callback if delivery.report.only.error=true */
+        if (self->u.Producer.dr_only_error && !rkm->err)
+                goto done;
+
 	msgobj = Message_new0(self, rkm);
 	
 	args = Py_BuildValue("(OO)",
diff --git a/confluent_kafka/src/confluent_kafka.c b/confluent_kafka/src/confluent_kafka.c
@@ -1124,7 +1124,26 @@ static int producer_conf_set_special (Handle *self, rd_kafka_conf_t *conf,
 		}
 
 		return 1;
-	}
+
+        } else if (!strcmp(name, "delivery.report.only.error")) {
+                /* Since we allocate msgstate for each produced message
+                 * with a callback we can't use delivery.report.only.error
+                 * as-is, as we wouldn't be able to ever free those msgstates.
+                 * Instead we shortcut this setting in the Python client,
+                 * providing the same functionality from dr_msg_cb trampoline.
+                 */
+
+                if (!PyBool_Check(valobj)) {
+                        cfl_PyErr_Format(
+                                RD_KAFKA_RESP_ERR__INVALID_ARG,
+                                "%s requires bool", name);
+                        return -1;
+                }
+
+                self->u.Producer.dr_only_error = valobj == Py_True;
+
+                return 1;
+        }
 
 	return 0; /* Not handled */
 }
diff --git a/confluent_kafka/src/confluent_kafka.h b/confluent_kafka/src/confluent_kafka.h
@@ -138,6 +138,8 @@ typedef struct {
 				const rd_kafka_topic_t *,
 				const void *, size_t, int32_t,
 				void *, void *);  /**< Fallback C partitioner*/
+
+                        int dr_only_error; /**< delivery.report.only.error */
 		} Producer;
 
 		/**
diff --git a/examples/integration_test.py b/examples/integration_test.py
@@ -25,6 +25,7 @@
 import uuid
 import sys
 import json
+import gc
 from copy import copy
 
 try:
@@ -134,6 +135,82 @@ def verify_producer():
     # Block until all messages are delivered/failed
     p.flush()
 
+    #
+    # Additional isolated tests
+    #
+    test_producer_dr_only_error()
+
+
+
+# Global variable to track garbage collection of suppressed on_delivery callbacks
+DrOnlyTestSuccess_gced = 0
+
+def test_producer_dr_only_error():
+    """
+    The C delivery.report.only.error configuration property
+    can't be used with the Python client since the Python client
+    allocates a msgstate for each produced message that has a callback,
+    and on success (with delivery.report.only.error=true) the delivery report
+    will not be called and the msgstate will thus never be freed.
+
+    Since a proper broker is required for messages to be succesfully sent
+    this test must be run from the integration tests rather than
+    the unit tests.
+    """
+    p = confluent_kafka.Producer({"bootstrap.servers": bootstrap_servers,
+                                  'broker.address.family':'v4',
+                                  "delivery.report.only.error":True})
+
+    class DrOnlyTestErr (object):
+        def __init__ (self):
+            self.remaining = 1
+
+        def handle_err (self, err, msg):
+            """ This delivery handler should only get called for errored msgs """
+            assert "BAD:" in msg.value().decode('utf-8')
+            assert err is not None
+            self.remaining -= 1
+
+    class DrOnlyTestSuccess (object):
+        def handle_success (self, err, msg):
+            """ This delivery handler should never get called """
+            # FIXME: Can we verify that it is actually garbage collected?
+            assert "GOOD:" in msg.value().decode('utf-8')
+            assert err is None
+            assert False, "should never come here"
+
+        def __del__ (self):
+            # Indicate that gc has hit this object.
+            global DrOnlyTestSuccess_gced
+            DrOnlyTestSuccess_gced = 1
+
+    print('only.error: Verifying delivery.report.only.error')
+
+    state = DrOnlyTestErr()
+    p.produce(topic, "BAD: This message will make not make it".encode('utf-8'),
+              partition=99, on_delivery=state.handle_err)
+
+    not_called_state = DrOnlyTestSuccess()
+    p.produce(topic, "GOOD: This message will make make it".encode('utf-8'),
+              on_delivery=not_called_state.handle_success)
+
+    # Garbage collection should not kick in yet for not_called_state
+    # since there is a on_delivery reference to it.
+    not_called_state = None
+    gc.collect()
+    global DrOnlyTestSuccess_gced
+    assert DrOnlyTestSuccess_gced == 0
+
+    print('only.error: Waiting for flush of %d messages' % len(p))
+    p.flush(10000)
+
+    print('only.error: Remaining messages now %d' % state.remaining)
+    assert state.remaining == 0
+
+    # Now with all messages flushed the reference to not_called_state should be gone.
+    gc.collect()
+    assert DrOnlyTestSuccess_gced == 1
+
 
 def verify_avro():
     from confluent_kafka import avro

Original file line number	Diff line number	Diff line change
`@@ -152,6 +152,10 @@ static void dr_msg_cb (rd_kafka_t rk, const rd_kafka_message_t rkm,`
`152`	`152`	`goto done;`
`153`	`153`	`}`
`154`	`154`
	`155`	`+ /* Skip callback if delivery.report.only.error=true */`
	`156`	`+ if (self->u.Producer.dr_only_error && !rkm->err)`
	`157`	`+ goto done;`
	`158`	`+`
`155`	`159`	`msgobj = Message_new0(self, rkm);`
`156`	`160`
`157`	`161`	`args = Py_BuildValue("(OO)",`