diff --git a/uncoder-core/app/translator/core/mixins/tokens.py b/uncoder-core/app/translator/core/mixins/tokens.py new file mode 100644 index 00000000..09bbe266 --- /dev/null +++ b/uncoder-core/app/translator/core/mixins/tokens.py @@ -0,0 +1,20 @@ +from typing import Union + +from app.translator.core.const import QUERY_TOKEN_TYPE +from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType +from app.translator.core.mapping import SourceMapping +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier + + +class ExtraConditionMixin: + def generate_extra_conditions(self, source_mapping: SourceMapping) -> list[QUERY_TOKEN_TYPE]: + extra_tokens = [] + for field, value in source_mapping.conditions.items(): + extra_tokens.extend( + [ + FieldValue(source_name=field, operator=Identifier(token_type=OperatorType.EQ), value=value), + Identifier(token_type=LogicalOperatorType.AND), + ] + ) + return extra_tokens diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 857c2516..673cc6fa 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -403,6 +403,9 @@ def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping if raw_log_field_type := source_mapping.raw_log_fields.get(field): return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)] + def generate_extra_conditions(self, source_mapping: SourceMapping) -> list[QUERY_TOKEN_TYPE]: # noqa: ARG002 + return [] + def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str: if not self.raw_log_field_patterns_map: return "" @@ -442,6 +445,9 @@ def _generate_from_tokenized_query_container_by_source_mapping( source_mapping=source_mapping, ) prefix += f"\n{defined_raw_log_fields}" + if source_mapping.conditions: + extra_tokens = self.generate_extra_conditions(source_mapping=source_mapping) + query_container.tokens = [*extra_tokens, *query_container.tokens] query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported return self.finalize_query( diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/default.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/default.yml new file mode 100644 index 00000000..ef7bc834 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/arcsight/default.yml @@ -0,0 +1,5 @@ +platform: ArcSight +source: default + + +default_log_source: {} diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/linux_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/linux_network_connection.yml new file mode 100644 index 00000000..d720251d --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/arcsight/linux_network_connection.yml @@ -0,0 +1,9 @@ +platform: ArcSight +source: linux_network_connection + + +default_log_source: {} + + +field_mapping: + SourceHostname: sourceHostName \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/macos_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/macos_network_connection.yml new file mode 100644 index 00000000..85370b92 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/arcsight/macos_network_connection.yml @@ -0,0 +1,9 @@ +platform: ArcSight +source: macos_network_connection + + +default_log_source: {} + + +field_mapping: + SourceHostname: sourceHostName \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_create_remote_thread.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_create_remote_thread.yml new file mode 100644 index 00000000..4b9f3e91 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_create_remote_thread.yml @@ -0,0 +1,13 @@ +platform: ArcSight +source: windows_create_remote_thread + + +default_log_source: {} + + +field_mapping: + SourceImage: sourceProcessName + TargetImage: destinationProcessName + StartModule: deviceCustomString3 + StartAddress: deviceCustomString3 + StartFunction: deviceCustomString3 \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_network_connection.yml new file mode 100644 index 00000000..32d52d69 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_network_connection.yml @@ -0,0 +1,9 @@ +platform: ArcSight +source: windows_network_connection + + +default_log_source: {} + + +field_mapping: + SourceHostname: sourceHostName \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_process_creation.yml new file mode 100644 index 00000000..356466c8 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_process_creation.yml @@ -0,0 +1,9 @@ +platform: ArcSight +source: windows_process_creation + + +default_log_source: {} + + +field_mapping: + OriginalFileName: oldFileName \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_security.yml new file mode 100644 index 00000000..57803a9b --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_security.yml @@ -0,0 +1,54 @@ +platform: ArcSight +source: windows_security + + +default_log_source: {} + +conditions: + deviceVendor: Microsoft + deviceProduct: Microsoft Windows + + +field_mapping: + EventID: externalId + AccessMask: deviceCustomString1 + AccountName: destinationUserName + AuditPolicyChanges: deviceAction + AuthenticationPackageName: deviceCustomString5 + EventType: deviceSeverity + FailureReason: deviceCustomString4 + IpAddress: sourceAddress + IpPort: sourcePort + LogonProcessName: + - destinationProcessName + - sourceProcessName + LogonType: deviceCustomNumber1 + MemberName: destinationUserId + MemberSid: destinationUserName + NewProcessName: destinationProcessName + ObjectClass: deviceCustomString5 + ObjectName: fileName + ObjectType: fileType + ObjectValueName: deviceCustomString6 + CommandLine: deviceCustomString4 + ProcessName: destinationProcessName + Properties: deviceCustomString6 + ServiceFileName: filePath + ServiceName: destinationServiceName + ShareName: + - filePath + - deviceCustomString6 + Status: eventOutcome + SubjectDomainName: destinationNTDomain + SubjectUserName: destinationUserName + SubjectUserSid: destinationUserName + TargetDomainName: destinationNTDomain + TargetSid: destinationNTDomain + TargetUserName: destinationUserName + TargetUserSid: destinationUserName + TicketEncryptionType: deviceCustomString5 + TicketOptions: deviceCustomString1 + WorkstationName: sourceHostName + ServiceType: fileType + StartType: deviceCustomString5 + ParentProcessName: filePath \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/arcsight/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_sysmon.yml new file mode 100644 index 00000000..e92d02a8 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/arcsight/windows_sysmon.yml @@ -0,0 +1,50 @@ +platform: ArcSight +source: windows_sysmon + + +default_log_source: {} + +conditions: + deviceVendor: Microsoft + deviceProduct: Sysmon + +field_mapping: + CommandLine: deviceCustomString1 + Image: destinationProcessName + ParentImage: sourceProcessName + EventID: externalId + CallTrace: deviceCustomString3 + Company: oldFileType + CurrentDirectory: deviceCustomString3 + Description: oldFilePermission + DestinationHostname: destinationHostName + DestinationIp: destinationAddress + DestinationPort: destinationPort + Initiated: deviceCustomString4 + IntegrityLevel: deviceCustomString5 + ParentCommandLine: deviceCustomString2 + Product: destinationServiceName + Protocol: transportProtocol + RuleName: deviceFacility + SourceHostname: sourceHostName + SourceIp: sourceAddress + SourcePort: sourcePort + TargetFilename: fileName + User: sourceUserName + OriginalFileName: oldFileName + Signed: deviceCustomString1 + Signature: deviceCustomString2 + SignatureStatus: deviceCustomString3 + TargetObject: fileName + Details: deviceCustomString1 + QueryName: + - requestUrl + - destinationHostName + QueryResults: deviceCustomString1 + QueryStatus: deviceCustomNumber1 + PipeName: fileName + ImageLoaded: destinationProcessName + SourceImage: sourceProcessName + StartModule: deviceCustomString3 + TargetImage: destinationProcessName + EventType: deviceAction \ No newline at end of file diff --git a/uncoder-core/app/translator/platforms/arcsight/__init__.py b/uncoder-core/app/translator/platforms/arcsight/__init__.py index cefce570..f666494e 100644 --- a/uncoder-core/app/translator/platforms/arcsight/__init__.py +++ b/uncoder-core/app/translator/platforms/arcsight/__init__.py @@ -1 +1,2 @@ +from app.translator.platforms.arcsight.renders.arcsight import ArcSightQueryRender # noqa: F401 from app.translator.platforms.arcsight.renders.arcsight_cti import ArcsightKeyword # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/arcsight/const.py b/uncoder-core/app/translator/platforms/arcsight/const.py index 0bd27667..f9af112e 100644 --- a/uncoder-core/app/translator/platforms/arcsight/const.py +++ b/uncoder-core/app/translator/platforms/arcsight/const.py @@ -1,8 +1,12 @@ +from app.translator.core.models.platform_details import PlatformDetails + ARCSIGHT_QUERY_DETAILS = { - "platform_id": "arcsight", + "platform_id": "arcsight-query", "name": "ArcSight Query", "group_name": "ArcSight", "group_id": "arcsight", "platform_name": "Query", "alt_platform_name": "CEF", } + +arcsight_query_details = PlatformDetails(**ARCSIGHT_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/arcsight/escape_manager.py b/uncoder-core/app/translator/platforms/arcsight/escape_manager.py new file mode 100644 index 00000000..6478e2ff --- /dev/null +++ b/uncoder-core/app/translator/platforms/arcsight/escape_manager.py @@ -0,0 +1,14 @@ +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.escape_manager import EscapeManager +from app.translator.core.models.escape_details import EscapeDetails + + +class ArcSightEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern='(["\\()])', escape_symbols="\\\\\g<1>")] + } + + +arcsight_escape_manager = ArcSightEscapeManager() diff --git a/uncoder-core/app/translator/platforms/arcsight/mapping.py b/uncoder-core/app/translator/platforms/arcsight/mapping.py new file mode 100644 index 00000000..b5686f48 --- /dev/null +++ b/uncoder-core/app/translator/platforms/arcsight/mapping.py @@ -0,0 +1,18 @@ +from app.translator.core.mapping import BaseStrictLogSourcesPlatformMappings, LogSourceSignature +from app.translator.platforms.arcsight.const import arcsight_query_details + + +class ArcSightLogSourceSignature(LogSourceSignature): + def is_suitable(self) -> bool: + return True + + def __str__(self) -> str: + return "" + + +class ArcSightMappings(BaseStrictLogSourcesPlatformMappings): + def prepare_log_source_signature(self, mapping: dict) -> ArcSightLogSourceSignature: # noqa: ARG002 + return ArcSightLogSourceSignature() + + +arcsight_query_mappings = ArcSightMappings(platform_dir="arcsight", platform_details=arcsight_query_details) diff --git a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight.py b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight.py new file mode 100644 index 00000000..3bb65d38 --- /dev/null +++ b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight.py @@ -0,0 +1,101 @@ +from typing import Optional, Union + +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.custom_types.values import ValueType +from app.translator.core.mapping import LogSourceSignature +from app.translator.core.mixins.tokens import ExtraConditionMixin +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.core.str_value_manager import StrValue, StrValueManager +from app.translator.managers import render_manager +from app.translator.platforms.arcsight.const import arcsight_query_details +from app.translator.platforms.arcsight.mapping import ArcSightMappings, arcsight_query_mappings +from app.translator.platforms.arcsight.str_value_manager import arcsight_str_value_manager + + +class ArcSightFieldValue(BaseFieldValueRender): + details: PlatformDetails = arcsight_query_details + str_value_manager: StrValueManager = arcsight_str_value_manager + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f'"{value}"' + + @staticmethod + def _wrap_int_value(value: int) -> str: + return f'"{value}"' + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.equal_modifier(field, val) for val in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True) + return f"{field} = {value}" + + def less_modifier(self, field: str, value: Union[int, str, StrValue]) -> str: + return f"{field} < {self._pre_process_value(field, value, wrap_str=True)}" + + def less_or_equal_modifier(self, field: str, value: Union[int, str, StrValue]) -> str: + return f"{field} <= {self._pre_process_value(field, value, wrap_str=True)}" + + def greater_modifier(self, field: str, value: Union[int, str, StrValue]) -> str: + return f"{field} > {self._pre_process_value(field, value, wrap_str=True)}" + + def greater_or_equal_modifier(self, field: str, value: Union[int, str, StrValue]) -> str: + return f"{field} > {self._pre_process_value(field, value, wrap_str=True)}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.not_equal_modifier(field, val) for val in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True) + return f"{field} != {value}" + + def is_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.is_none(field=field, value=v) for v in value)})" + return f"NOT _exists_:{field}" + + def is_not_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.is_not_none(field=field, value=v) for v in value)})" + return f"_exists_:{field}" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.contains_modifier(field, val) for val in value)})" + value = self._wrap_str_value(value) + return f"{field} CONTAINS {value}" + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.endswith_modifier(field, val) for val in value)})" + value = self._wrap_str_value(value) + return f"{field} ENDSWITH {value}" + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.startswith_modifier(field, val) for val in value)})" + value = self._wrap_str_value(value) + return f"{field} STARTSWITH {value}" + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field, val) for val in value)})" + value = self._wrap_str_value(value) + return f"{field} CONTAINS {value}" + + +@render_manager.register +class ArcSightQueryRender(ExtraConditionMixin, PlatformQueryRender): + details: PlatformDetails = arcsight_query_details + mappings: ArcSightMappings = arcsight_query_mappings + + or_token = "OR" + and_token = "AND" + not_token = "NOT" + + comment_symbol = "//" + + field_value_render = ArcSightFieldValue(or_token=or_token) + + def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002 + return "" diff --git a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py index 778ef04e..8e41b677 100644 --- a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py +++ b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py @@ -1,13 +1,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render_cti import RenderCTI from app.translator.managers import render_cti_manager -from app.translator.platforms.arcsight.const import ARCSIGHT_QUERY_DETAILS +from app.translator.platforms.arcsight.const import arcsight_query_details from app.translator.platforms.arcsight.mappings.arcsight_cti import DEFAULT_ARCSIGHT_MAPPING @render_cti_manager.register class ArcsightKeyword(RenderCTI): - details: PlatformDetails = PlatformDetails(**ARCSIGHT_QUERY_DETAILS) + details: PlatformDetails = arcsight_query_details default_mapping = DEFAULT_ARCSIGHT_MAPPING field_value_template: str = "{key} = {value}" diff --git a/uncoder-core/app/translator/platforms/arcsight/str_value_manager.py b/uncoder-core/app/translator/platforms/arcsight/str_value_manager.py new file mode 100644 index 00000000..e9a98b2a --- /dev/null +++ b/uncoder-core/app/translator/platforms/arcsight/str_value_manager.py @@ -0,0 +1,27 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from app.translator.core.str_value_manager import StrValueManager +from app.translator.platforms.arcsight.escape_manager import ArcSightEscapeManager, arcsight_escape_manager + + +class ArcSightStrValueManager(StrValueManager): + escape_manager: ArcSightEscapeManager = arcsight_escape_manager + + +arcsight_str_value_manager = ArcSightStrValueManager() diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py index f3a6b937..8fe19cd8 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py @@ -2,12 +2,11 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.const import QUERY_TOKEN_TYPE -from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType +from app.translator.core.custom_types.tokens import GroupType from app.translator.core.custom_types.values import ValueType -from app.translator.core.mapping import LogSourceSignature, SourceMapping +from app.translator.core.mapping import LogSourceSignature +from app.translator.core.mixins.tokens import ExtraConditionMixin from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.models.query_container import TokenizedQueryContainer -from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValueManager @@ -105,7 +104,6 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_int=True) - return f'{self.apply_field(field)} regex~ "{value}.?"' def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -121,7 +119,8 @@ def is_not_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: AR @render_manager.register -class ElasticSearchEQLQueryRender(PlatformQueryRender): + +class ElasticSearchEQLQueryRender(ExtraConditionMixin, PlatformQueryRender): details: PlatformDetails = elastic_eql_query_details mappings: LuceneMappings = elastic_eql_query_mappings or_token = "or" @@ -135,41 +134,4 @@ def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], fu def in_brackets(self, raw_list: list[QUERY_TOKEN_TYPE]) -> list[QUERY_TOKEN_TYPE]: return [Identifier(token_type=GroupType.L_PAREN), *raw_list, Identifier(token_type=GroupType.R_PAREN)] - - def _generate_from_tokenized_query_container_by_source_mapping( - self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping - ) -> str: - unmapped_fields = self.mappings.check_fields_mapping_existence( - query_container.meta_info.query_fields, - query_container.meta_info.function_fields_map, - self.platform_functions.manager.supported_render_names, - source_mapping, - ) - rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) - prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix) - - if source_mapping.raw_log_fields: - defined_raw_log_fields = self.generate_raw_log_fields( - fields=query_container.meta_info.query_fields + query_container.meta_info.function_fields, - source_mapping=source_mapping, - ) - prefix += f"\n{defined_raw_log_fields}" - if source_mapping.conditions: - for field, value in source_mapping.conditions.items(): - tokens = self.in_brackets(query_container.tokens) - extra_tokens = [ - FieldValue(source_name=field, operator=Identifier(token_type=OperatorType.EQ), value=value), - Identifier(token_type=LogicalOperatorType.AND), - ] - query_container.tokens = self.in_brackets([*extra_tokens, *tokens]) - query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) - not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported - return self.finalize_query( - prefix=prefix, - query=query, - functions=rendered_functions.rendered, - not_supported_functions=not_supported_functions, - unmapped_fields=unmapped_fields, - meta_info=query_container.meta_info, - source_mapping=source_mapping, - ) + pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy