Python-Kafka logging package

addu390 · addu390 · commit e4dabadffdf8 · 2020-10-02T12:22:23.000+05:30
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+.idea
+__pycache__
+*.pyc
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,3 @@
+include *.txt
+include *.md
+include *.rst
diff --git a/README.md b/README.md
diff --git a/python_logging_kafka/__init__.py b/python_logging_kafka/__init__.py
diff --git a/python_logging_kafka/consumer.py b/python_logging_kafka/consumer.py
@@ -0,0 +1,4 @@
+from kafka import KafkaConsumer
+consumer = KafkaConsumer('pyblog')
+for message in consumer:
+    print (message)
diff --git a/python_logging_kafka/handlers.py b/python_logging_kafka/handlers.py
@@ -0,0 +1,34 @@
+from logging import StreamHandler
+from kafka import KafkaProducer
+
+
+class KafkaHandler(StreamHandler):
+
+    def __init__(self, broker, topic):
+        StreamHandler.__init__(self)
+        self.broker = broker
+        self.topic = topic
+
+        self.kafka_broker = MyKafka(broker)
+
+    def emit(self, record):
+        message = self.format(record)
+        self.kafka_broker.send(message, self.topic)
+
+
+class MyKafka(object):
+
+    def __init__(self, kafka_brokers, json=False):
+        self.json = json
+        if not json:
+            self.producer = KafkaProducer(bootstrap_servers=kafka_brokers)
+        else:
+            self.producer = KafkaProducer(value_serializer=lambda v: json.dumps(v).encode('utf-8'),
+                                          bootstrap_servers=kafka_brokers)
+
+    def send(self, data, topic):
+        if self.json:
+            result = self.producer.send(topic, key=b'log', value=data)
+        else:
+            result = self.producer.send(topic, bytes(data, 'utf-8'))
+        print("kafka send result: {}".format(result.get()))
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1 @@
+kafka-python==2.0.1
diff --git a/setup.py b/setup.py
@@ -0,0 +1,39 @@
+import pathlib
+from setuptools import setup
+
+HERE = pathlib.Path(__file__).parent
+README = (HERE / "README.md").read_text()
+
+CLASSIFIERS = [
+    'Intended Audience :: Developers',
+    'License :: OSI Approved :: MIT License',
+    'Operating System :: OS Independent',
+    'Programming Language :: Python',
+    'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
+]
+
+INSTALL_REQUIRES = [
+    'kafka-python>=2.0.1'
+]
+
+TEST_REQUIRES = [
+    "pytest>=4.3.0",
+]
+
+setup(name='python-logging-kafka',
+      version='1.0.0',
+      url='https://github.com/albertomr86/python-logging-rabbitmq',
+      description='Kafka log producer from Python/Django',
+      long_description=README,
+      keywords='logging kafka logs',
+      license='MIT',
+      author='Adesh Nalpet',
+      author_email="390.adesh@gmail.com",
+      classifiers=CLASSIFIERS,
+      python_requires="!=3.5.*, !=3.6.*, !=3.7.*",
+      install_requires=INSTALL_REQUIRES,
+      include_package_data=True,
+      packages=['python_logging_kafka'],
+      test_suite='tests',
+      zip_safe=True)
diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/test.py b/tests/test.py
@@ -0,0 +1,2 @@
+def mock_test():
+    assert True == True

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+include *.txt`
	`2`	`+include *.md`
	`3`	`+include *.rst`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+def mock_test():`
	`2`	`+ assert True == True`