Merge pull request larymak#302 from Jerry0420/add-multiprocessing-logger

larymak · web-flow · commit 964b9572946a · 2023-06-22T09:23:55.000+03:00
add multiprocessing logging helper
diff --git a/OTHERS/MultiprocessingLogger/README.md b/OTHERS/MultiprocessingLogger/README.md
@@ -0,0 +1,28 @@
+# Python Logging Helper for Multiple Processes
+
+## Description
+This is a helper class that allows Python developers to log to a single log file from multiple processes easily.
+
+## Examples
+The following is an example code snippet in **main.py**, which provides two modes:
+
+1. Running in Multiprocessing Pool:
+```python
+python3 main.py --is_pool yes
+```
+
+1. Running in Normal Multiprocessing:
+```python
+python3 main.py --is_pool no
+```
+
+The log file will be created in the log directory and named mulp.log.
+
+* Log File Content
+```
+06-17 13:41:13 | sub | INFO | /projects/OTHERS/MultiprocessingLogger/main.py:19 | 26802 | c
+06-17 13:41:13 | sub | INFO | /projects/OTHERS/MultiprocessingLogger/main.py:19 | 26803 | d
+06-17 13:41:13 | sub | INFO | /projects/OTHERS/MultiprocessingLogger/main.py:19 | 26799 | a
+06-17 13:41:13 | sub | INFO | /projects/OTHERS/MultiprocessingLogger/main.py:19 | 26804 | e
+06-17 13:41:13 | sub | INFO | /projects/OTHERS/MultiprocessingLogger/main.py:19 | 26801 | b
+```
diff --git a/OTHERS/MultiprocessingLogger/logger.py b/OTHERS/MultiprocessingLogger/logger.py
@@ -0,0 +1,84 @@
+import logging
+import logging.handlers
+import multiprocessing
+import os
+import sys
+
+from pathlib import Path
+
+class MultiProcesses_Logging_Helper:
+
+    def __init__(self) -> None:
+        self.queue = multiprocessing.Manager().Queue(-1)
+        
+        self.process = multiprocessing.get_context('fork').Process(
+            target=self._process_target,
+        )
+        self.process.start()
+        self.redirect_main_process_log_to_queue()
+
+    def get_log_file_path(self) -> str:
+        # TODO: make this configurable
+        log_file_dir = os.path.join(Path(__file__).parent, 'logs')
+        Path(log_file_dir).mkdir(parents=True, exist_ok=True)
+        log_file_path = os.path.join(log_file_dir, 'multip.log')
+        return log_file_path
+
+    def redirect_main_process_log_to_queue(self):
+
+        main_process_id = os.getpid()
+
+        def filter(record: logging.LogRecord):
+            if record.process == main_process_id:
+                self.queue.put_nowait(record)
+            return None
+
+        root = logging.getLogger()
+        root.setLevel(logging.INFO)
+
+        handler = logging.Handler()
+        handler.addFilter(filter)
+        root.addHandler(handler)
+
+    def init_logger_configure(self):
+        
+        root = logging.getLogger()
+        root.setLevel(logging.INFO)
+
+        # TODO: make this configurable
+        file_handler = logging.handlers.RotatingFileHandler(
+            self.get_log_file_path(),
+            mode='w',
+            # maxBytes=5000,
+            # backupCount=0,
+            encoding='utf-8'
+        )
+        file_handler.setLevel(logging.INFO)
+        file_handler_formatter = logging.Formatter(
+            fmt='%(asctime)s | %(name)s | %(levelname)s | %(pathname)s:%(lineno)d | %(process)d | %(message)s',
+            datefmt="%m-%d %H:%M:%S"
+        )
+        file_handler.setFormatter(file_handler_formatter)
+        root.addHandler(file_handler)
+
+        std_handler = logging.StreamHandler(sys.stdout)
+        std_handler.setLevel(logging.DEBUG)
+        std_handler_formatter = logging.Formatter(
+            fmt='%(asctime)s | %(name)s | %(levelname)s | %(pathname)s:%(lineno)d | %(process)d | %(message)s',
+            datefmt="%m-%d %H:%M:%S"
+        )
+        std_handler.setFormatter(std_handler_formatter)
+        root.addHandler(std_handler)
+
+    def _process_target(self):
+        self.init_logger_configure()
+        while True:
+            record: logging.LogRecord = self.queue.get()
+            if record is None:
+                break
+            logger = logging.getLogger(record.name)
+            logger.handle(record)
+
+    def close(self):
+        self.process.join(timeout=0)
+        self.process.terminate()
diff --git a/OTHERS/MultiprocessingLogger/main.py b/OTHERS/MultiprocessingLogger/main.py
@@ -0,0 +1,52 @@
+import argparse
+import logging
+import logging.handlers
+import multiprocessing
+from functools import partial
+from logger import MultiProcesses_Logging_Helper
+
+
+def process_target(queue: multiprocessing.Queue, input: str):
+    # add queue handler at the beginning of the process
+    sub_logger = logging.getLogger('sub')
+    sub_logger.setLevel(logging.INFO)
+    queue_handler = logging.handlers.QueueHandler(queue)
+
+    # this process may be reused multiple times (when multiprocessing pool) 
+    if len(sub_logger.handlers) == 0 or not isinstance(sub_logger.handlers[0], logging.handlers.QueueHandler):
+        sub_logger.addHandler(queue_handler)
+
+    sub_logger.info(input)
+
+
+def main(is_pool: bool):
+    logger_helper = MultiProcesses_Logging_Helper()
+
+    if is_pool:
+        print("run in multiprocessing pool")
+        pool = multiprocessing.Pool(2)
+        inputs = [i for i in ["a", "b", "c", "d", "e"]]
+        pool.map(partial(process_target, logger_helper.queue), inputs)
+        logger_helper.queue.put_nowait(None)
+        pool.close()
+        pool.join()
+    else:    
+        print("run in multiprocessing")
+        workers = []
+        for i in ["a", "b", "c", "d", "e"]:
+            worker = multiprocessing.get_context('fork').Process(target=process_target,
+                                                args=(logger_helper.queue, i))
+            workers.append(worker)
+            worker.start()
+        for w in workers:
+            w.join()
+        logger_helper.queue.put_nowait(None)
+
+    logger_helper.close()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='')
+    parser.add_argument('-p', '--is_pool', help='is run in multiprocessing pool mode', default=True, type=lambda x: (str(x).lower() in ['true','1', 'yes', 'y', 'True']))
+    args = parser.parse_args()
+    main(args.is_pool)
diff --git a/README.md b/README.md
@@ -120,4 +120,5 @@ guide [HERE](https://github.com/larymak/Python-project-Scripts/blob/main/CONTRIB
 | 69    | [Bitcoin Price](https://github.com/larymak/Python-project-Scripts/tree/main/WEB%20SCRAPING/Bitcoin%20Price)                                                                | [Olu-Olagbuji Delight](https://github.com/Dheelyte)           
 | 70    | [Password Generator](https://github.com/larymak/Python-project-Scripts/tree/main/GUI/Password%20Generator)                                                                | [LpCodes](https://github.com/LpCodes)     
 | 71    | [HTML to Excel](https://github.com/larymak/Python-project-Scripts/tree/main/CONVERSION%20SCRIPTS/HTML%20to%20Excel)                                                                | [LpCodes](https://github.com/LpCodes)     
-| 72    | [Star pattern](https://github.com/larymak/Python-project-Scripts/tree/main/OTHERS/Star%20pattern)                                                                | [LpCodes](https://github.com/LpCodes)     
+| 72    | [Star pattern](https://github.com/larymak/Python-project-Scripts/tree/main/OTHERS/Star%20pattern)                                                                | [LpCodes](https://github.com/LpCodes)     |
+| 73    | [Logging Helper](https://github.com/larymak/Python-project-Scripts/tree/main/OTHERS/add-multiprocessing-logger)                                                                | [Jerry W.](https://github.com/Jerry0420)     |
