Added support for fatal errors

edenhill · edenhill · commit b1ec10f936db · 2018-11-15T10:51:07.000+01:00
diff --git a/confluent_kafka/kafkatest/verifiable_client.py b/confluent_kafka/kafkatest/verifiable_client.py
@@ -91,10 +91,6 @@ def set_config(conf, args):
                 # "org.apache.kafka.clients.consumer.RangeAssignor" -> "range"
                 conf[n] = re.sub(r'org.apache.kafka.clients.consumer.(\w+)Assignor',
                                  lambda x: x.group(1).lower(), v)
-
-            elif n == 'enable.idempotence':
-                # Ignore idempotence for now, best-effortly.
-                sys.stderr.write('%% WARN: Ignoring unsupported %s=%s\n' % (n, v))
             else:
                 conf[n] = v
 
diff --git a/confluent_kafka/src/confluent_kafka.c b/confluent_kafka/src/confluent_kafka.c
@@ -62,9 +62,12 @@ typedef struct {
 	char *str;   /* Human readable representation of error, if one
 		      * was provided by librdkafka.
 		      * Else falls back on err2str(). */
+        int   fatal; /**< Set to true if a fatal error. */
 } KafkaError;
 
 
+static void cfl_PyErr_Fatal (rd_kafka_resp_err_t err, const char *reason);
+
 static PyObject *KafkaError_code (KafkaError *self, PyObject *ignore) {
 	return cfl_PyInt_FromInt(self->code);
 }
@@ -81,6 +84,20 @@ static PyObject *KafkaError_name (KafkaError *self, PyObject *ignore) {
 	return cfl_PyUnistr(_FromString(rd_kafka_err2name(self->code)));
 }
 
+static PyObject *KafkaError_fatal (KafkaError *self, PyObject *ignore) {
+        PyObject *ret = self->fatal ? Py_True : Py_False;
+        Py_INCREF(ret);
+        return ret;
+}
+
+
+static PyObject *KafkaError_test_raise_fatal (KafkaError *null,
+                                              PyObject *ignore) {
+        cfl_PyErr_Fatal(RD_KAFKA_RESP_ERR__INVALID_ARG,
+                        "This is a fatal exception for testing purposes");
+        return NULL;
+}
+
 
 static PyMethodDef KafkaError_methods[] = {
 	{ "code", (PyCFunction)KafkaError_code, METH_NOARGS,
@@ -105,6 +122,14 @@ static PyMethodDef KafkaError_methods[] = {
 	  "  :rtype: str\n"
 	  "\n"
 	},
+        { "fatal", (PyCFunction)KafkaError_fatal, METH_NOARGS,
+          "  :returns: True if this a fatal error, else False.\n"
+          "  :rtype: bool\n"
+          "\n"
+        },
+        { "_test_raise_fatal", (PyCFunction)KafkaError_test_raise_fatal,
+          METH_NOARGS|METH_STATIC
+        },
 
 	{ NULL }
 };
@@ -133,7 +158,8 @@ static int KafkaError_traverse (KafkaError *self,
 }
 
 static PyObject *KafkaError_str0 (KafkaError *self) {
-	return cfl_PyUnistr(_FromFormat("KafkaError{code=%s,val=%d,str=\"%s\"}",
+	return cfl_PyUnistr(_FromFormat("KafkaError{%scode=%s,val=%d,str=\"%s\"}",
+                                        self->fatal?"FATAL,":"",
 					 rd_kafka_err2name(self->code),
 					 self->code,
 					 self->str ? self->str :
@@ -247,6 +273,7 @@ static PyTypeObject KafkaErrorType = {
 static void KafkaError_init (KafkaError *self,
 			     rd_kafka_resp_err_t code, const char *str) {
 	self->code = code;
+        self->fatal = 0;
 	if (str)
 		self->str = strdup(str);
 	else
@@ -292,6 +319,17 @@ PyObject *KafkaError_new0 (rd_kafka_resp_err_t err, const char *fmt, ...) {
 }
 
 
+/**
+ * @brief Raise exception from fatal error.
+ */
+static void cfl_PyErr_Fatal (rd_kafka_resp_err_t err, const char *reason) {
+        PyObject *eo = KafkaError_new0(err, "%s", reason);
+        ((KafkaError *)eo)->fatal = 1;
+        PyErr_SetObject(KafkaException, eo);
+}
+
+
+
 
 
 /****************************************************************************
@@ -1162,6 +1200,7 @@ PyObject *c_headers_to_py (rd_kafka_headers_t *headers) {
 }
 #endif
 
+
 /****************************************************************************
  *
  *
@@ -1177,6 +1216,16 @@ static void error_cb (rd_kafka_t *rk, int err, const char *reason, void *opaque)
 	CallState *cs;
 
 	cs = CallState_get(h);
+
+        /* If the client raised a fatal error we'll raise an exception
+         * rather than calling the error callback. */
+        if (err == RD_KAFKA_RESP_ERR__FATAL) {
+                char errstr[512];
+                err = rd_kafka_fatal_error(rk, errstr, sizeof(errstr));
+                cfl_PyErr_Fatal(err, errstr);
+                goto crash;
+        }
+
 	if (!h->error_cb) {
 		/* No callback defined */
 		goto done;
@@ -1189,6 +1238,7 @@ static void error_cb (rd_kafka_t *rk, int err, const char *reason, void *opaque)
 	if (result)
 		Py_DECREF(result);
 	else {
+        crash:
 		CallState_crash(cs);
 		rd_kafka_yield(h->rk);
 	}
@@ -1766,8 +1816,7 @@ rd_kafka_conf_t *common_conf_setup (rd_kafka_type_t ktype,
 
         Py_DECREF(confdict);
 
-	if (h->error_cb)
-		rd_kafka_conf_set_error_cb(conf, error_cb);
+        rd_kafka_conf_set_error_cb(conf, error_cb);
 
         if (h->throttle_cb)
                 rd_kafka_conf_set_throttle_cb(conf, throttle_cb);
diff --git a/tests/test_KafkaError.py b/tests/test_KafkaError.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import pytest
 from confluent_kafka import Producer, KafkaError, KafkaException
 import time
 
@@ -8,6 +9,7 @@
 
 def error_cb(err):
     print('error_cb', err)
+    assert err.fatal() is False
     if err.code() == KafkaError._ALL_BROKERS_DOWN:
         global seen_all_brokers_down
         seen_all_brokers_down = True
@@ -31,6 +33,22 @@ def test_error_cb():
     assert seen_all_brokers_down
 
 
+def test_fatal():
+    """ Test fatal exceptions """
+
+    # Configure an invalid broker and make sure the ALL_BROKERS_DOWN
+    # error is seen in the error callback.
+    p = Producer({'error_cb': error_cb})
+
+    with pytest.raises(KafkaException) as exc:
+        KafkaError._test_raise_fatal()
+    err = exc.value.args[0]
+    assert isinstance(err, KafkaError)
+    assert err.fatal() is True
+
+    p.poll(0)  # Need some p use to avoid flake8 unused warning
+
+
 def test_subclassing():
     class MyExc(KafkaException):
         def a_method(self):
