Merge branch 'main' into gis-8825

nazargesyk · web-flow · commit 4dff1e13463f · 2024-12-17T15:21:19.000+02:00
diff --git a/uncoder-core/app/translator/core/mixins/tokens.py b/uncoder-core/app/translator/core/mixins/tokens.py
@@ -0,0 +1,20 @@
+from typing import Union
+
+from app.translator.core.const import QUERY_TOKEN_TYPE
+from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType
+from app.translator.core.mapping import SourceMapping
+from app.translator.core.models.query_tokens.field_value import FieldValue
+from app.translator.core.models.query_tokens.identifier import Identifier
+
+
+class ExtraConditionMixin:
+    def generate_extra_conditions(self, source_mapping: SourceMapping) -> list[QUERY_TOKEN_TYPE]:
+        extra_tokens = []
+        for field, value in source_mapping.conditions.items():
+            extra_tokens.extend(
+                [
+                    FieldValue(source_name=field, operator=Identifier(token_type=OperatorType.EQ), value=value),
+                    Identifier(token_type=LogicalOperatorType.AND),
+                ]
+            )
+        return extra_tokens
diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py
@@ -403,6 +403,9 @@ def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping
         if raw_log_field_type := source_mapping.raw_log_fields.get(field):
             return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)]
 
+    def generate_extra_conditions(self, source_mapping: SourceMapping) -> list[QUERY_TOKEN_TYPE]:  # noqa: ARG002
+        return []
+
     def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str:
         if not self.raw_log_field_patterns_map:
             return ""
@@ -442,6 +445,9 @@ def _generate_from_tokenized_query_container_by_source_mapping(
                 source_mapping=source_mapping,
             )
             prefix += f"\n{defined_raw_log_fields}"
+        if source_mapping.conditions:
+            extra_tokens = self.generate_extra_conditions(source_mapping=source_mapping)
+            query_container.tokens = [*extra_tokens, *query_container.tokens]
         query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
         not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported
         return self.finalize_query(
diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/default.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/default.yml
@@ -0,0 +1,5 @@
+platform: ArcSight
+source: default
+
+
+default_log_source: {}
diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/linux_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/linux_network_connection.yml
@@ -0,0 +1,9 @@
+platform: ArcSight
+source: linux_network_connection
+
+
+default_log_source: {}
+
+
+field_mapping:
+  SourceHostname: sourceHostName
diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/macos_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/macos_network_connection.yml
@@ -0,0 +1,9 @@
+platform: ArcSight
+source: macos_network_connection
+
+
+default_log_source: {}
+
+
+field_mapping:
+  SourceHostname: sourceHostName
diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_create_remote_thread.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_create_remote_thread.yml
@@ -0,0 +1,13 @@
+platform: ArcSight
+source: windows_create_remote_thread
+
+
+default_log_source: {}
+
+
+field_mapping:
+  SourceImage: sourceProcessName
+  TargetImage: destinationProcessName
+  StartModule: deviceCustomString3
+  StartAddress: deviceCustomString3
+  StartFunction: deviceCustomString3
diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_network_connection.yml
@@ -0,0 +1,9 @@
+platform: ArcSight
+source: windows_network_connection
+
+
+default_log_source: {}
+
+
+field_mapping:
+  SourceHostname: sourceHostName
diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_process_creation.yml
@@ -0,0 +1,9 @@
+platform: ArcSight
+source: windows_process_creation
+
+
+default_log_source: {}
+
+
+field_mapping:
+  OriginalFileName: oldFileName
diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_security.yml
@@ -0,0 +1,54 @@
+platform: ArcSight
+source: windows_security
+
+
+default_log_source: {}
+
+conditions:
+  deviceVendor: Microsoft
+  deviceProduct: Microsoft Windows
+
+
+field_mapping:
+  EventID: externalId
+  AccessMask: deviceCustomString1
+  AccountName: destinationUserName
+  AuditPolicyChanges: deviceAction
+  AuthenticationPackageName: deviceCustomString5
+  EventType: deviceSeverity
+  FailureReason: deviceCustomString4
+  IpAddress: sourceAddress
+  IpPort: sourcePort
+  LogonProcessName:
+    - destinationProcessName
+    - sourceProcessName
+  LogonType: deviceCustomNumber1
+  MemberName: destinationUserId
+  MemberSid: destinationUserName
+  NewProcessName: destinationProcessName
+  ObjectClass: deviceCustomString5
+  ObjectName: fileName
+  ObjectType: fileType
+  ObjectValueName: deviceCustomString6
+  CommandLine: deviceCustomString4
+  ProcessName: destinationProcessName
+  Properties: deviceCustomString6
+  ServiceFileName: filePath
+  ServiceName: destinationServiceName
+  ShareName:
+    - filePath
+    - deviceCustomString6
+  Status: eventOutcome
+  SubjectDomainName: destinationNTDomain
+  SubjectUserName: destinationUserName
+  SubjectUserSid: destinationUserName
+  TargetDomainName: destinationNTDomain
+  TargetSid: destinationNTDomain
+  TargetUserName: destinationUserName
+  TargetUserSid: destinationUserName
+  TicketEncryptionType: deviceCustomString5
+  TicketOptions: deviceCustomString1
+  WorkstationName: sourceHostName
+  ServiceType: fileType
+  StartType: deviceCustomString5
+  ParentProcessName: filePath
diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_sysmon.yml
@@ -0,0 +1,50 @@
+platform: ArcSight
+source: windows_sysmon
+
+
+default_log_source: {}
+
+conditions:
+  deviceVendor: Microsoft
+  deviceProduct: Sysmon
+
+field_mapping:
+  CommandLine: deviceCustomString1
+  Image: destinationProcessName
+  ParentImage: sourceProcessName
+  EventID: externalId
+  CallTrace: deviceCustomString3
+  Company: oldFileType
+  CurrentDirectory: deviceCustomString3
+  Description: oldFilePermission
+  DestinationHostname: destinationHostName
+  DestinationIp: destinationAddress
+  DestinationPort: destinationPort
+  Initiated: deviceCustomString4
+  IntegrityLevel: deviceCustomString5
+  ParentCommandLine: deviceCustomString2
+  Product: destinationServiceName
+  Protocol: transportProtocol
+  RuleName: deviceFacility
+  SourceHostname: sourceHostName
+  SourceIp: sourceAddress
+  SourcePort: sourcePort
+  TargetFilename: fileName
+  User: sourceUserName
+  OriginalFileName: oldFileName
+  Signed: deviceCustomString1
+  Signature: deviceCustomString2
+  SignatureStatus: deviceCustomString3
+  TargetObject: fileName
+  Details: deviceCustomString1
+  QueryName:
+    - requestUrl
+    - destinationHostName
+  QueryResults: deviceCustomString1
+  QueryStatus: deviceCustomNumber1
+  PipeName: fileName
+  ImageLoaded: destinationProcessName
+  SourceImage: sourceProcessName
+  StartModule: deviceCustomString3
+  TargetImage: destinationProcessName
+  EventType: deviceAction
diff --git a/uncoder-core/app/translator/platforms/arcsight/__init__.py b/uncoder-core/app/translator/platforms/arcsight/__init__.py
@@ -1 +1,2 @@
+from app.translator.platforms.arcsight.renders.arcsight import ArcSightQueryRender  # noqa: F401
 from app.translator.platforms.arcsight.renders.arcsight_cti import ArcsightKeyword  # noqa: F401
diff --git a/uncoder-core/app/translator/platforms/arcsight/const.py b/uncoder-core/app/translator/platforms/arcsight/const.py
@@ -1,8 +1,12 @@
+from app.translator.core.models.platform_details import PlatformDetails
+
 ARCSIGHT_QUERY_DETAILS = {
-    "platform_id": "arcsight",
+    "platform_id": "arcsight-query",
     "name": "ArcSight Query",
     "group_name": "ArcSight",
     "group_id": "arcsight",
     "platform_name": "Query",
     "alt_platform_name": "CEF",
 }
+
+arcsight_query_details = PlatformDetails(**ARCSIGHT_QUERY_DETAILS)
diff --git a/uncoder-core/app/translator/platforms/arcsight/escape_manager.py b/uncoder-core/app/translator/platforms/arcsight/escape_manager.py
@@ -0,0 +1,14 @@
+from typing import ClassVar
+
+from app.translator.core.custom_types.values import ValueType
+from app.translator.core.escape_manager import EscapeManager
+from app.translator.core.models.escape_details import EscapeDetails
+
+
+class ArcSightEscapeManager(EscapeManager):
+    escape_map: ClassVar[dict[str, list[EscapeDetails]]] = {
+        ValueType.value: [EscapeDetails(pattern='(["\\()])', escape_symbols="\\\\\g<1>")]
+    }
+
+
+arcsight_escape_manager = ArcSightEscapeManager()
diff --git a/uncoder-core/app/translator/platforms/arcsight/mapping.py b/uncoder-core/app/translator/platforms/arcsight/mapping.py
@@ -0,0 +1,18 @@
+from app.translator.core.mapping import BaseStrictLogSourcesPlatformMappings, LogSourceSignature
+from app.translator.platforms.arcsight.const import arcsight_query_details
+
+
+class ArcSightLogSourceSignature(LogSourceSignature):
+    def is_suitable(self) -> bool:
+        return True
+
+    def __str__(self) -> str:
+        return ""
+
+
+class ArcSightMappings(BaseStrictLogSourcesPlatformMappings):
+    def prepare_log_source_signature(self, mapping: dict) -> ArcSightLogSourceSignature:  # noqa: ARG002
+        return ArcSightLogSourceSignature()
+
+
+arcsight_query_mappings = ArcSightMappings(platform_dir="arcsight", platform_details=arcsight_query_details)
diff --git a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight.py b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight.py
@@ -0,0 +1,101 @@
+from typing import Optional, Union
+
+from app.translator.const import DEFAULT_VALUE_TYPE
+from app.translator.core.custom_types.values import ValueType
+from app.translator.core.mapping import LogSourceSignature
+from app.translator.core.mixins.tokens import ExtraConditionMixin
+from app.translator.core.models.platform_details import PlatformDetails
+from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
+from app.translator.core.str_value_manager import StrValue, StrValueManager
+from app.translator.managers import render_manager
+from app.translator.platforms.arcsight.const import arcsight_query_details
+from app.translator.platforms.arcsight.mapping import ArcSightMappings, arcsight_query_mappings
+from app.translator.platforms.arcsight.str_value_manager import arcsight_str_value_manager
+
+
+class ArcSightFieldValue(BaseFieldValueRender):
+    details: PlatformDetails = arcsight_query_details
+    str_value_manager: StrValueManager = arcsight_str_value_manager
+
+    @staticmethod
+    def _wrap_str_value(value: str) -> str:
+        return f'"{value}"'
+
+    @staticmethod
+    def _wrap_int_value(value: int) -> str:
+        return f'"{value}"'
+
+    def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.equal_modifier(field, val) for val in value)})"
+        value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)
+        return f"{field} = {value}"
+
+    def less_modifier(self, field: str, value: Union[int, str, StrValue]) -> str:
+        return f"{field} < {self._pre_process_value(field, value, wrap_str=True)}"
+
+    def less_or_equal_modifier(self, field: str, value: Union[int, str, StrValue]) -> str:
+        return f"{field} <= {self._pre_process_value(field, value, wrap_str=True)}"
+
+    def greater_modifier(self, field: str, value: Union[int, str, StrValue]) -> str:
+        return f"{field} > {self._pre_process_value(field, value, wrap_str=True)}"
+
+    def greater_or_equal_modifier(self, field: str, value: Union[int, str, StrValue]) -> str:
+        return f"{field} > {self._pre_process_value(field, value, wrap_str=True)}"
+
+    def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.not_equal_modifier(field, val) for val in value)})"
+        value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)
+        return f"{field} != {value}"
+
+    def is_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.is_none(field=field, value=v) for v in value)})"
+        return f"NOT _exists_:{field}"
+
+    def is_not_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.is_not_none(field=field, value=v) for v in value)})"
+        return f"_exists_:{field}"
+
+    def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.contains_modifier(field, val) for val in value)})"
+        value = self._wrap_str_value(value)
+        return f"{field} CONTAINS {value}"
+
+    def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.endswith_modifier(field, val) for val in value)})"
+        value = self._wrap_str_value(value)
+        return f"{field} ENDSWITH {value}"
+
+    def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.startswith_modifier(field, val) for val in value)})"
+        value = self._wrap_str_value(value)
+        return f"{field} STARTSWITH {value}"
+
+    def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.regex_modifier(field, val) for val in value)})"
+        value = self._wrap_str_value(value)
+        return f"{field} CONTAINS {value}"
+
+
+@render_manager.register
+class ArcSightQueryRender(ExtraConditionMixin, PlatformQueryRender):
+    details: PlatformDetails = arcsight_query_details
+    mappings: ArcSightMappings = arcsight_query_mappings
+
+    or_token = "OR"
+    and_token = "AND"
+    not_token = "NOT"
+
+    comment_symbol = "//"
+
+    field_value_render = ArcSightFieldValue(or_token=or_token)
+
+    def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str:  # noqa: ARG002
+        return ""
diff --git a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py
@@ -1,13 +1,13 @@
 from app.translator.core.models.platform_details import PlatformDetails
 from app.translator.core.render_cti import RenderCTI
 from app.translator.managers import render_cti_manager
-from app.translator.platforms.arcsight.const import ARCSIGHT_QUERY_DETAILS
+from app.translator.platforms.arcsight.const import arcsight_query_details
 from app.translator.platforms.arcsight.mappings.arcsight_cti import DEFAULT_ARCSIGHT_MAPPING
 
 
 @render_cti_manager.register
 class ArcsightKeyword(RenderCTI):
-    details: PlatformDetails = PlatformDetails(**ARCSIGHT_QUERY_DETAILS)
+    details: PlatformDetails = arcsight_query_details
 
     default_mapping = DEFAULT_ARCSIGHT_MAPPING
     field_value_template: str = "{key} = {value}"
diff --git a/uncoder-core/app/translator/platforms/arcsight/str_value_manager.py b/uncoder-core/app/translator/platforms/arcsight/str_value_manager.py
diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py
