UncoderIO · nazargesyk · Dec 17, 2024 · Nov 14, 2024 · Nov 14, 2024 · Nov 14, 2024
diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py
@@ -65,6 +65,8 @@ def __init__(
         date: Optional[str] = None,
         output_table_fields: Optional[list[Field]] = None,
         query_fields: Optional[list[Field]] = None,
+        function_fields: Optional[list[Field]] = None,
+        function_fields_map: Optional[dict[str, list[Field]]] = None,
         license_: Optional[str] = None,
         severity: Optional[str] = None,
         references: Optional[list[str]] = None,
@@ -76,7 +78,7 @@ def __init__(
         parsed_logsources: Optional[dict] = None,
         timeframe: Optional[timedelta] = None,
         query_period: Optional[timedelta] = None,
-        mitre_attack: MitreInfoContainer = MitreInfoContainer(),
+        mitre_attack: Optional[MitreInfoContainer] = None,
         raw_metainfo_container: Optional[RawMetaInfoContainer] = None,
     ) -> None:
         self.id = id_ or str(uuid.uuid4())
@@ -86,23 +88,25 @@ def __init__(
         self.risk_score = risk_score
         self.type_ = type_ or ""
         self.description = description or ""
-        self.author = [v.strip() for v in author] if author else []
+        self.author = [v.strip() for v in author] if author and author != [None] else []
         self.date = date or datetime.now().date().strftime("%Y-%m-%d")
         self.output_table_fields = output_table_fields or []
         self.query_fields = query_fields or []
+        self.function_fields = function_fields or []
+        self.function_fields_map = function_fields_map or {}
         self.license = license_ or "DRL 1.1"
         self.severity = severity or SeverityType.low
         self.references = references or []
         self.tags = tags or []
-        self.mitre_attack = mitre_attack or None
+        self.mitre_attack = mitre_attack or MitreInfoContainer()
         self.raw_mitre_attack = raw_mitre_attack or []
         self.status = status or "stable"
         self.false_positives = false_positives or []
         self._source_mapping_ids = source_mapping_ids or [DEFAULT_MAPPING_NAME]
         self.parsed_logsources = parsed_logsources or {}
         self.timeframe = timeframe
         self.query_period = query_period
-        self.raw_metainfo_container = raw_metainfo_container
+        self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer()
 
     @property
     def author_str(self) -> str:

diff --git a/uncoder-core/app/translator/platforms/microsoft/const.py b/uncoder-core/app/translator/platforms/microsoft/const.py
@@ -19,15 +19,18 @@
 
 PLATFORM_DETAILS = {"group_id": "sentinel", "group_name": "Microsoft Sentinel"}
 
+_SENTINEL_KQL_QUERY = "sentinel-kql-query"
+_SENTINEL_KQL_RULE = "sentinel-kql-rule"
+
 MICROSOFT_SENTINEL_QUERY_DETAILS = {
-    "platform_id": "sentinel-kql-query",
+    "platform_id": _SENTINEL_KQL_QUERY,
     "name": "Microsoft Sentinel Query",
     "platform_name": "Query (Kusto)",
     **PLATFORM_DETAILS,
 }
 
 MICROSOFT_SENTINEL_RULE_DETAILS = {
-    "platform_id": "sentinel-kql-rule",
+    "platform_id": _SENTINEL_KQL_RULE,
     "name": "Microsoft Sentinel Rule",
     "platform_name": "Rule (Kusto)",
     "first_choice": 0,
@@ -50,6 +53,8 @@
     "group_id": "microsoft-defender",
 }
 
+MICROSOFT_SENTINEL_QUERY_TYPES = {_SENTINEL_KQL_QUERY, _SENTINEL_KQL_RULE}
+
 microsoft_defender_query_details = PlatformDetails(**MICROSOFT_DEFENDER_DETAILS)
 microsoft_sentinel_query_details = PlatformDetails(**MICROSOFT_SENTINEL_QUERY_DETAILS)
 microsoft_sentinel_rule_details = PlatformDetails(**MICROSOFT_SENTINEL_RULE_DETAILS)

diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py
@@ -22,6 +22,7 @@
 from app.translator.const import DEFAULT_VALUE_TYPE
 from app.translator.core.mapping import LogSourceSignature
 from app.translator.core.models.platform_details import PlatformDetails
+from app.translator.core.models.query_container import RawQueryContainer
 from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
 from app.translator.managers import render_manager
 from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details
@@ -144,3 +145,6 @@ def generate_prefix(self, log_source_signature: LogSourceSignature, functions_pr
     @staticmethod
     def _finalize_search_query(query: str) -> str:
         return f"| where {query}" if query else ""
+
+    def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str:
+        return query_container.query
diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py
@@ -26,7 +26,7 @@
 from app.translator.core.custom_types.meta_info import SeverityType
 from app.translator.core.mapping import SourceMapping
 from app.translator.core.models.platform_details import PlatformDetails
-from app.translator.core.models.query_container import MetaInfoContainer, MitreInfoContainer
+from app.translator.core.models.query_container import MetaInfoContainer, MitreInfoContainer, RawQueryContainer
 from app.translator.managers import render_manager
 from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details
 from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_rule_mappings
@@ -107,7 +107,8 @@ def finalize_query(
         *args,  # noqa: ARG002
         **kwargs,  # noqa: ARG002
     ) -> str:
-        query = super().finalize_query(prefix=prefix, query=query, functions=functions)
+        if not kwargs.get("raw_query", False):
+            query = super().finalize_query(prefix=prefix, query=query, functions=functions)
         rule = copy.deepcopy(DEFAULT_MICROSOFT_SENTINEL_RULE)
         rule["query"] = query
         rule["displayName"] = meta_info.title or _AUTOGENERATED_TEMPLATE
@@ -130,3 +131,8 @@ def finalize_query(
         json_rule = json.dumps(rule, indent=4, sort_keys=False)
         json_rule = self.wrap_with_unmapped_fields(json_rule, unmapped_fields)
         return self.wrap_with_not_supported_functions(json_rule, not_supported_functions)
+
+    def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str:
+        return self.finalize_query(
+            prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info, raw_query=True
+        )
diff --git a/uncoder-core/app/translator/translator.py b/uncoder-core/app/translator/translator.py
@@ -1,12 +1,16 @@
 import logging
-from typing import Optional
+from collections import Counter
+from typing import Optional, Union
 
 from app.translator.core.exceptions.core import UnsupportedPlatform
 from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
-from app.translator.core.parser import QueryParser
+from app.translator.core.parser import PlatformQueryParser, QueryParser
 from app.translator.core.render import QueryRender
 from app.translator.managers import ParserManager, RenderManager, parser_manager, render_manager
 from app.translator.platforms.elasticsearch.const import ELASTIC_QUERY_TYPES
+from app.translator.platforms.microsoft.const import MICROSOFT_SENTINEL_QUERY_TYPES
+from app.translator.platforms.roota.parsers.roota import RootAParser
+from app.translator.platforms.sigma.mapping import sigma_rule_mappings
 from app.translator.tools.decorators import handle_translation_exceptions
 
 
@@ -32,18 +36,36 @@ def __get_render(self, target: str) -> QueryRender:
 
     @staticmethod
     def __is_one_vendor_translation(source: str, target: str) -> bool:
-        vendors_query_types = [ELASTIC_QUERY_TYPES]
+        vendors_query_types = [ELASTIC_QUERY_TYPES, MICROSOFT_SENTINEL_QUERY_TYPES]
         for vendor_query_types in vendors_query_types:
             if source in vendor_query_types and target in vendor_query_types:
                 return True
 
         return False
 
-    def parse_raw_query(self, text: str, source: str) -> tuple[QueryParser, RawQueryContainer]:
+    def parse_raw_query(
+        self, text: str, source: str
+    ) -> tuple[Union[PlatformQueryParser, RootAParser], RawQueryContainer]:
         parser = self.__get_parser(source)
         text = parser.remove_comments(text)
         return parser, parser.parse_raw_query(text, language=source)
 
+    def parse_meta_info(self, text: str, source: str) -> Union[dict, RawQueryContainer]:
+        parser, raw_query_container = self.parse_raw_query(text=text, source=source)
+        source_mappings = parser.get_source_mapping_ids_by_logsources(raw_query_container.query)
+        log_sources = {"product": Counter(), "service": Counter(), "category": Counter()}
+        sigma_source_mappings = sigma_rule_mappings.get_source_mappings_by_ids(
+            [source_mapping.source_id for source_mapping in source_mappings], return_default=False
+        )
+        for sigma_source_mapping in sigma_source_mappings:
+            if product := sigma_source_mapping.log_source_signature.log_sources.get("product"):
+                log_sources["product"][product] += 1
+            if service := sigma_source_mapping.log_source_signature.log_sources.get("service"):
+                log_sources["service"][service] += 1
+            if category := sigma_source_mapping.log_source_signature.log_sources.get("category"):
+                log_sources["category"][category] += 1
+        return log_sources, raw_query_container
+
     @handle_translation_exceptions
     def __parse_incoming_data(
         self, text: str, source: str, target: Optional[str] = None