diff --git a/uncoder-core/app/routers/translate.py b/uncoder-core/app/routers/translate.py index 009cab03..b1f3867d 100644 --- a/uncoder-core/app/routers/translate.py +++ b/uncoder-core/app/routers/translate.py @@ -3,23 +3,29 @@ from app.models.translation import InfoMessage, OneTranslationData, Platform, TranslatorPlatforms from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.cti_translator import CTITranslator -from app.translator.translator import Translator +from app.translator.translator import app_translator st_router = APIRouter() -translator = Translator() - @st_router.post("/translate", tags=["translator"], description="Generate target translation") @st_router.post("/translate/", include_in_schema=False) def translate_one( source_platform_id: str = Body(..., embed=True), + source_alt_mapping: dict = Body(None, embed=True), target_platform_id: str = Body(..., embed=True), + target_alt_mapping: dict = Body(None, embed=True), text: str = Body(..., embed=True), return_only_first_query: bool = False, ) -> OneTranslationData: return_only_first_query_ctx_var.set(return_only_first_query) - status, data = translator.translate_one(text=text, source=source_platform_id, target=target_platform_id) + status, data = app_translator.translate_one( + text=text, + source=source_platform_id, + target=target_platform_id, + source_alt_mapping=source_alt_mapping, + target_alt_mapping=target_alt_mapping, + ) if status: return OneTranslationData(status=status, translation=data, target_platform_id=target_platform_id) @@ -35,7 +41,7 @@ def translate_all( return_only_first_query: bool = False, ) -> list[OneTranslationData]: return_only_first_query_ctx_var.set(return_only_first_query) - result = translator.translate_all(text=text, source=source_platform_id) + result = app_translator.translate_all(text=text, source=source_platform_id) translations = [] for platform_result in result: if platform_result.get("status"): @@ -60,14 +66,14 @@ def translate_all( @st_router.get("/platforms", tags=["translator"], description="Get translator platforms") @st_router.get("/platforms/", include_in_schema=False) def get_translator_platforms() -> TranslatorPlatforms: - renders, parsers = translator.get_all_platforms() + renders, parsers = app_translator.get_all_platforms() return TranslatorPlatforms(renders=renders, parsers=parsers) @st_router.get("/all_platforms", description="Get Sigma, RootA and iocs platforms") @st_router.get("/all_platforms/", include_in_schema=False) def get_all_platforms() -> list: - translator_renders, translator_parsers = translator.get_all_platforms() + translator_renders, translator_parsers = app_translator.get_all_platforms() return [ Platform( id="roota", diff --git a/uncoder-core/app/translator/core/exceptions/core.py b/uncoder-core/app/translator/core/exceptions/core.py index 425c1ff0..3ded7f24 100644 --- a/uncoder-core/app/translator/core/exceptions/core.py +++ b/uncoder-core/app/translator/core/exceptions/core.py @@ -42,6 +42,14 @@ def __init__(self, platform: str, is_parser: bool = False): super().__init__(message) +class UnsupportedPlatformAlternativeMapping(BasePlatformException): + def __init__(self, platform: str, alt_mapping: str): + message = ( + f"The selected data schema '{alt_mapping}' doesn't have a field mapping for this log source in {platform}." + ) + super().__init__(message) + + class UnsupportedRootAParser(BasePlatformException): def __init__(self, parser: str): message = ( diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 1c4d2070..3bacd6af 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -1,9 +1,14 @@ from __future__ import annotations +import os from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Optional, TypeVar, Union +from typing import TYPE_CHECKING, ClassVar, Optional, TypeVar, Union -from app.translator.core.exceptions.core import StrictPlatformException, UnsupportedMappingsException +from app.translator.core.exceptions.core import ( + StrictPlatformException, + UnsupportedMappingsException, + UnsupportedPlatformAlternativeMapping, +) from app.translator.core.models.platform_details import PlatformDetails from app.translator.mappings.utils.load_from_files import LoaderFileMappings @@ -12,6 +17,7 @@ DEFAULT_MAPPING_NAME = "default" +GLOBAL_ALTERNATIVE_MAPPING_DIR = "global_alternative" class LogSourceSignature(ABC): @@ -109,24 +115,39 @@ class BasePlatformMappings: is_strict_mapping: bool = False skip_load_default_mappings: bool = True extend_default_mapping_with_all_fields: bool = False + global_alternative_mappings: ClassVar[list[str]] = [] def __init__(self, platform_dir: str, platform_details: PlatformDetails): self._loader = LoaderFileMappings() - self._platform_dir = platform_dir self.details = platform_details - self._source_mappings = self.prepare_mapping() + self._source_mappings = self.prepare_mapping(platform_dir) + self._alternative_mappings = self.prepare_alternative_mapping(platform_dir) + if self.global_alternative_mappings: + self._alternative_mappings.update(self.prepare_global_alternative_mapping()) def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None: default_mapping.fields_mapping.update(fields_mapping) - def prepare_mapping(self) -> dict[str, SourceMapping]: + def prepare_alternative_mapping(self, platform_dir: str) -> dict[str, dict[str, SourceMapping]]: + alternative_mappings = {} + for name, platform_dir in self._loader.get_platform_alternative_mappings_dirs(platform_dir).items(): + alternative_mappings[name] = self.prepare_mapping(platform_dir) + return alternative_mappings + + def prepare_global_alternative_mapping(self) -> dict[str, dict[str, SourceMapping]]: + global_alternative_mappings = {} + for name in self.global_alternative_mappings: + global_alternative_mappings[name] = self.prepare_mapping(os.path.join(GLOBAL_ALTERNATIVE_MAPPING_DIR, name)) + return global_alternative_mappings + + def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: source_mappings = {} default_mapping = SourceMapping(source_id=DEFAULT_MAPPING_NAME) - for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): + for mapping_dict in self._loader.load_platform_mappings(platform_dir): log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) if (source_id := mapping_dict["source"]) == DEFAULT_MAPPING_NAME: default_mapping.log_source_signature = log_source_signature - if self.skip_load_default_mappings: + if self.skip_load_default_mappings and not platform_dir.startswith(GLOBAL_ALTERNATIVE_MAPPING_DIR): continue field_mappings_dict = mapping_dict.get("field_mapping", {}) @@ -143,7 +164,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]: conditions=conditions, ) - if self.skip_load_default_mappings: + if self.skip_load_default_mappings and not platform_dir.startswith(GLOBAL_ALTERNATIVE_MAPPING_DIR): source_mappings[DEFAULT_MAPPING_NAME] = default_mapping if self.extend_default_mapping_with_all_fields: @@ -164,10 +185,18 @@ def prepare_log_source_signature(self, mapping: dict) -> LogSourceSignature: def get_source_mappings_by_fields_and_log_sources( self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]] + ) -> list[SourceMapping]: + return self._get_source_mappings_by_fields_and_log_sources(field_names, log_sources, self._source_mappings) + + def _get_source_mappings_by_fields_and_log_sources( + self, + field_names: list[str], + log_sources: dict[str, list[Union[int, str]]], + source_mappings: dict[str, SourceMapping], ) -> list[SourceMapping]: by_log_sources_and_fields = [] by_fields = [] - for source_mapping in self._source_mappings.values(): + for source_mapping in source_mappings.values(): if source_mapping.source_id == DEFAULT_MAPPING_NAME: continue @@ -178,11 +207,23 @@ def get_source_mappings_by_fields_and_log_sources( if log_source_signature and log_source_signature.is_suitable(**log_sources): by_log_sources_and_fields.append(source_mapping) - return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]] + return by_log_sources_and_fields or by_fields or [source_mappings[DEFAULT_MAPPING_NAME]] + + def get_alt_source_mappings_by_fields_and_log_sources( + self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]], alt_mapping: str + ) -> list[SourceMapping]: + return self._get_source_mappings_by_fields_and_log_sources( + field_names, log_sources, self._alternative_mappings.get(alt_mapping, {}) + ) def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]: return self._source_mappings.get(source_id) + def get_alternative_source_mapping(self, alt_config_name: str, source_id: str) -> Optional[SourceMapping]: + if self._alternative_mappings.get(alt_config_name): + return self._alternative_mappings.get(alt_config_name).get(source_id) + raise UnsupportedPlatformAlternativeMapping(platform=self.details.platform_id, alt_mapping=alt_config_name) + def get_source_mappings_by_ids( self, source_mapping_ids: list[str], return_default: bool = True ) -> list[SourceMapping]: @@ -198,6 +239,21 @@ def get_source_mappings_by_ids( return source_mappings + def get_alternative_source_mappings_by_ids( + self, source_mapping_ids: list[str], alt_config_name: str, return_default: bool = True + ) -> list[SourceMapping]: + source_mappings = [] + for source_mapping_id in source_mapping_ids: + if source_mapping_id == DEFAULT_MAPPING_NAME: + continue + if source_mapping := self.get_alternative_source_mapping(alt_config_name, source_mapping_id): + source_mappings.append(source_mapping) + + if not source_mappings and return_default: + source_mappings = [self.get_alternative_source_mapping(alt_config_name, DEFAULT_MAPPING_NAME)] + + return source_mappings + def get_source_mappings_by_log_sources(self, log_sources: dict) -> Optional[list[str]]: raise NotImplementedError("Abstract method") @@ -249,11 +305,11 @@ def map_field(field: Field, source_mapping: SourceMapping) -> list[str]: class BaseCommonPlatformMappings(ABC, BasePlatformMappings): - def prepare_mapping(self) -> dict[str, SourceMapping]: + def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: source_mappings = {} - common_field_mapping = self._loader.load_common_mapping(self._platform_dir).get("field_mapping", {}) + common_field_mapping = self._loader.load_common_mapping(platform_dir).get("field_mapping", {}) - for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): + for mapping_dict in self._loader.load_platform_mappings(platform_dir): source_id = mapping_dict["source"] log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) fields_mapping = self.prepare_fields_mapping(field_mapping=common_field_mapping) diff --git a/uncoder-core/app/translator/core/mixins/tokens.py b/uncoder-core/app/translator/core/mixins/tokens.py index 09bbe266..2bba49c7 100644 --- a/uncoder-core/app/translator/core/mixins/tokens.py +++ b/uncoder-core/app/translator/core/mixins/tokens.py @@ -1,5 +1,3 @@ -from typing import Union - from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.mapping import SourceMapping diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index ad866b51..854c2972 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -75,6 +75,8 @@ def __init__( status: Optional[str] = None, false_positives: Optional[list[str]] = None, source_mapping_ids: Optional[list[str]] = None, + source_alt_mapping: Optional[str] = None, + target_alt_mapping: Optional[str] = None, parsed_logsources: Optional[dict] = None, timeframe: Optional[timedelta] = None, query_period: Optional[timedelta] = None, @@ -107,6 +109,8 @@ def __init__( self.timeframe = timeframe self.query_period = query_period self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer() + self.source_alt_mapping = source_alt_mapping + self.target_alt_mapping = target_alt_mapping @property def author_str(self) -> str: @@ -132,7 +136,7 @@ class RawQueryContainer: class RawQueryDictContainer: query: dict language: str - meta_info: dict + meta_info: MetaInfoContainer = field(default_factory=MetaInfoContainer) @dataclass diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index da7330eb..69eae9b1 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -24,7 +24,7 @@ from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping -from app.translator.core.models.functions.base import Function +from app.translator.core.models.functions.base import Function, ParsedFunctions from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.models.query_tokens.field import Field @@ -51,6 +51,9 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: raise NotImplementedError("Abstract method") + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + raise NotImplementedError("Abstract method") + class PlatformQueryParser(QueryParser, ABC): mappings: BasePlatformMappings = None @@ -80,11 +83,24 @@ def get_field_tokens( return query_field_tokens, function_field_tokens, function_field_tokens_map def get_source_mappings( - self, field_tokens: list[Field], log_sources: dict[str, list[Union[int, str]]] + self, + field_tokens: list[Field], + log_sources: dict[str, list[Union[int, str]]], + alt_mapping: Optional[str] = None, ) -> list[SourceMapping]: field_names = [field.source_name for field in field_tokens] - source_mappings = self.mappings.get_source_mappings_by_fields_and_log_sources( - field_names=field_names, log_sources=log_sources - ) + if alt_mapping: + source_mappings = self.mappings.get_alt_source_mappings_by_fields_and_log_sources( + field_names=field_names, log_sources=log_sources, alt_mapping=alt_mapping + ) + else: + source_mappings = self.mappings.get_source_mappings_by_fields_and_log_sources( + field_names=field_names, log_sources=log_sources + ) self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping) return source_mappings + + def get_source_mapping_ids_by_logsources(self, query: str) -> Optional[list[str]]: + _, parsed_logsources, _ = self._parse_query(query=query) + if parsed_logsources: + return self.mappings.get_source_mappings_by_log_sources(parsed_logsources) diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 673cc6fa..32cab632 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -463,7 +463,15 @@ def _generate_from_tokenized_query_container_by_source_mapping( def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} errors = [] - source_mappings = self.mappings.get_source_mappings_by_ids(query_container.meta_info.source_mapping_ids) + if query_container.meta_info.target_alt_mapping: + source_mappings = self.mappings.get_alternative_source_mappings_by_ids( + source_mapping_ids=query_container.meta_info.source_mapping_ids, + alt_config_name=query_container.meta_info.target_alt_mapping, + ) + else: + source_mappings = self.mappings.get_source_mappings_by_ids( + source_mapping_ids=query_container.meta_info.source_mapping_ids + ) for source_mapping in source_mappings: try: diff --git a/uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml b/uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml new file mode 100644 index 00000000..fad3735c --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml @@ -0,0 +1,433 @@ +platform: Global OCSF +source: default + + +field_mapping: + IntegrityLevel: process.integrity + ParentCommandLine: + - process.parent_process.cmd_line + - actor.process.cmd_line + ParentUser: process.parent_process.user + Product: + - process.file.product.name + - process.file.product.vendor_name + TargetFilename: file.name + OriginalFileName: module.file.path + TargetObject: + - reg_key.path + - reg_value.name + - reg_value.path + AdditionalEventData: unmapped.additionalEventData + additionalEventData.MFAUsed: unmapped.additionalEventData.MFAUsed + eventType: unmapped.eventType + responseElements: unmapped.responseElements + responseElements.ConsoleLogin: unmapped.responseElements.ConsoleLogin + userAgent: http_request.user_agent + userIdentity.arn: unmapped.userIdentity_sessionContext_sessionIssuer_arn + userIdentity.principalId: unmapped.userIdentity_sessionContext_sessionIssuer_principalId + userIdentity.sessionContext.sessionIssuer.type: unmapped.userIdentity_sessionContext_sessionIssuer_type + userIdentity.type: identity.user.type + userIdentity.userName: unmapped.userIdentity_sessionContext_sessionIssuer_userName + a0: unmapped.a0 + a1: unmapped.a1 + a2: unmapped.a2 + a3: unmapped.a3 + AADOperationType: unmapped.AADOperationType + Accesses: unmapped.Accesses + AccessMask: unmapped.AccessMask + AccountName: unmapped.AccountName + action: unmapped.action + action_id: unmapped.action_id + ActionType: unmapped.ActionType + ActivityDisplayName: unmapped.ActivityDisplayName + ActivityInsights: unmapped.ActivityInsights + ActivityStatus: unmapped.ActivityStatus + ActivityStatusValue: unmapped.ActivityStatusValue + ActivitySubstatusValue: unmapped.ActivitySubstatusValue + ActivityType: unmapped.ActivityType + additional_information: unmapped.additional_information + Address: unmapped.Address + AllowedToDelegateTo: unmapped.AllowedToDelegateTo + alternateId: unmapped.alternateId + answers: unmapped.answers + AppDisplayName: unmapped.AppDisplayName + AppId: unmapped.AppId + ApplicationPath: unmapped.ApplicationPath + attackName: unmapped.attackName + AttributeLDAPDisplayName: unmapped.AttributeLDAPDisplayName + AttributeValue: unmapped.AttributeValue + AuditPolicyChanges: unmapped.AuditPolicyChanges + AuditSourceName: unmapped.AuditSourceName + AuthenticationAlgorithm: unmapped.AuthenticationAlgorithm + AuthenticationPackageName: unmapped.AuthenticationPackageName + AuthenticationRequirement: unmapped.AuthenticationRequirement + AuthenticationType: unmapped.AuthenticationType + Authorization: unmapped.Authorization + aws_node_type: unmapped.aws_node_type + backupConfiguration.enabled: unmapped.backupConfiguration.enabled + BSSID: unmapped.BSSID + BSSType: unmapped.BSSType + c-uri: unmapped.c-uri + c-uri-extension: unmapped.c-uri-extension + c-uri-query: unmapped.c-uri-query + c-useragent: unmapped.c-useragent + CallerIpAddress: unmapped.CallerIpAddress + CallerProcessName: unmapped.CallerProcessName + CallingProcessName: unmapped.CallingProcessName + CallTrace: unmapped.CallTrace + Caption: unmapped.Caption + category: unmapped.category + CategoryValue: unmapped.CategoryValue + certificate.serial: unmapped.certificate.serial + CertIssuerName: unmapped.CertIssuerName + CertSerialNumber: unmapped.CertSerialNumber + CertThumbprint: unmapped.CertThumbprint + Channel: unmapped.Channel + cipher: unmapped.cipher + CipherAlgorithm: unmapped.CipherAlgorithm + class_type: unmapped.class_type + ClassName: unmapped.ClassName + client: unmapped.client + client_header_names: unmapped.client_header_names + client.as.number: unmapped.client.as.number + client.as.organization.name: unmapped.client.as.organization.name + client.device: unmapped.client.device + client.domain: unmapped.client.domain + client.id: unmapped.client.id + client.ip: unmapped.client.ip + client.user.full_name: unmapped.client.user.full_name + client.user.id: unmapped.client.user.id + client.zone: unmapped.client.zone + ClientInfoString: unmapped.ClientInfoString + ClientProcessId: unmapped.ClientProcessId + CommandLine: unmapped.CommandLine + ComplianceState: unmapped.ComplianceState + ComputerName: unmapped.ComputerName + ConditionalAccessStatus: unmapped.ConditionalAccessStatus + conn_state: unmapped.conn_state + ConnectionId: unmapped.ConnectionId + ConnectionMode: unmapped.ConnectionMode + CONSUMER: unmapped.CONSUMER + cookie: unmapped.cookie + cs-bytes: unmapped.cs-bytes + cs-cookie: unmapped.cs-cookie + cs-cookie-vars: unmapped.cs-cookie-vars + cs-host: unmapped.cs-host + cs-method: unmapped.cs-method + cs-referrer: unmapped.cs-referrer + cs-version: unmapped.cs-version + CurrentDirectory: unmapped.CurrentDirectory + Data: unmapped.Data + data.protoPayload.methodName: unmapped.data.protoPayload.methodName + data.protoPayload.request.function.timeout: unmapped.data.protoPayload.request.function.timeout + data.protoPayload.serviceData.policyDelta.bindingDeltas{}.action: unmapped.data.protoPayload.serviceData.policyDelta.bindingDeltas{}.action + data.protoPayload.serviceData.policyDelta.bindingDeltas{}.member: unmapped.data.protoPayload.serviceData.policyDelta.bindingDeltas{}.member + data.resource.type: unmapped.data.resource.type + Description: unmapped.Description + desktop_height: unmapped.desktop_height + desktop_width: unmapped.desktop_width + DestAddress: unmapped.DestAddress + Destination: unmapped.Destination + DestinationHostname: unmapped.DestinationHostname + DestinationIp: unmapped.DestinationIp + DestinationPort: unmapped.DestinationPort + DestPort: unmapped.DestPort + Details: unmapped.Details + Device: unmapped.Device + DeviceDescription: unmapped.DeviceDescription + DeviceDetail: unmapped.DeviceDetail + DeviceName: unmapped.DeviceName + DevicesInsights: unmapped.DevicesInsights + DisableIntegrityChecks: unmapped.DisableIntegrityChecks + DisplayName: unmapped.DisplayName + dns-answer: unmapped.dns-answer + dns-query: unmapped.dns-query + dns-record: unmapped.dns-record + DnsHostName: unmapped.DnsHostName + Domain: unmapped.Domain + dst_ip: unmapped.dst_ip + dst_port: unmapped.dst_port + dst-ip: unmapped.dst-ip + dst-port: unmapped.dst-port + EnabledPrivilegeList: unmapped.EnabledPrivilegeList + endpoint: unmapped.endpoint + errorCode: unmapped.errorCode + errorMessage: unmapped.errorMessage + event.action: unmapped.event.action + event.outcome: unmapped.event.outcome + EventData: unmapped.EventData + EventID: unmapped.EventID + eventName: unmapped.eventName + eventSource: unmapped.eventSource + EventType: unmapped.EventType + exe: unmapped.exe + FailureReason: unmapped.FailureReason + Faultingapplicationpath: unmapped.Faultingapplicationpath + fc_request: unmapped.fc_request + FileName: unmapped.FileName + filetype: unmapped.filetype + function: unmapped.function + gcp.audit.method_name: unmapped.gcp.audit.method_name + GrantedAccess: unmapped.GrantedAccess + has_cert_table: unmapped.has_cert_table + Hashes: unmapped.Hashes + history: unmapped.history + HiveName: unmapped.HiveName + host: unmapped.host + Hostname: unmapped.Hostname + http.bodyMagic: unmapped.http.bodyMagic + http.method: unmapped.http.method + id_resp_p: unmapped.id_resp_p + id.orig_h: unmapped.id.orig_h + id.orig_p: unmapped.id.orig_p + id.resp_h: unmapped.id.resp_h + id.resp_p: unmapped.id.resp_p + Image: unmapped.Image + ImageLoaded: unmapped.ImageLoaded + ImageName: unmapped.ImageName + ImagePath: unmapped.ImagePath + inference: unmapped.inference + Initiated: unmapped.Initiated + InitiatedBy: unmapped.InitiatedBy + InterfaceDescription: unmapped.InterfaceDescription + InterfaceGuid: unmapped.InterfaceGuid + IpAddress: unmapped.IpAddress + IpPort: unmapped.IpPort + IsInteractive: unmapped.IsInteractive + Issuer: unmapped.Issuer + jsonPayload.enforcedSecurityPolicy.configuredAction: unmapped.jsonPayload.enforcedSecurityPolicy.configuredAction + jsonPayload.enforcedSecurityPolicy.matchedFieldValue: unmapped.jsonPayload.enforcedSecurityPolicy.matchedFieldValue + KernelDebug: unmapped.KernelDebug + keyboard_layout: unmapped.keyboard_layout + KeyLength: unmapped.KeyLength + LayerRTID: unmapped.LayerRTID + Level: unmapped.Level + LinkName: unmapped.LinkName + local_orig: unmapped.local_orig + local_resp: unmapped.local_resp + LocalName: unmapped.LocalName + LogEntry: unmapped.LogEntry + logged_ssh_inference: unmapped.logged_ssh_inference + LoggedByService: unmapped.LoggedByService + LogonError: unmapped.LogonError + LogonProcessName: unmapped.LogonProcessName + LogonType: unmapped.LogonType + MemberName: unmapped.MemberName + MemberSid: unmapped.MemberSid + message: unmapped.message + method: unmapped.method + methodName: unmapped.methodName + mime_type: unmapped.mime_type + mime_types: unmapped.mime_types + ModifiedProperties: unmapped.ModifiedProperties + ModifyingApplication: unmapped.ModifyingApplication + name: unmapped.name + Namespace: unmapped.Namespace + NetworkLocationDetails: unmapped.NetworkLocationDetails + NewProcessName: unmapped.NewProcessName + NewSd: unmapped.NewSd + NewTargetUserName: unmapped.NewTargetUserName + NewTemplateContent: unmapped.NewTemplateContent + NewUacValue: unmapped.NewUacValue + NewValue: unmapped.NewValue + note: unmapped.note + notice.cipher: unmapped.notice.cipher + notice.password: unmapped.notice.password + notice.request_type: unmapped.notice.request_type + notice.user_agent: unmapped.notice.user_agent + NotificationPackageName: unmapped.NotificationPackageName + object_name: unmapped.object_name + ObjectClass: unmapped.ObjectClass + ObjectDN: unmapped.ObjectDN + ObjectName: unmapped.ObjectName + objectRef.namespace: unmapped.objectRef.namespace + objectRef.resource: unmapped.objectRef.resource + objectRef.subresource: unmapped.objectRef.subresource + ObjectServer: unmapped.ObjectServer + ObjectType: unmapped.ObjectType + ObjectValueName: unmapped.ObjectValueName + OfficeObjectId: unmapped.OfficeObjectId + OfficeWorkload: unmapped.OfficeWorkload + OldTargetUserName: unmapped.OldTargetUserName + OldUacValue: unmapped.OldUacValue + OldValue: unmapped.OldValue + OnexEnabled: unmapped.OnexEnabled + operation: unmapped.operation + OperationName: unmapped.OperationName + OperationNameValue: unmapped.OperationNameValue + OperationType: unmapped.OperationType + orig_bytes: unmapped.orig_bytes + Origin: unmapped.Origin + outcome.reason: unmapped.outcome.reason + param1: unmapped.param1 + param2: unmapped.param2 + Parameters: unmapped.Parameters + parent-domain: unmapped.parent-domain + ParentImage: unmapped.ParentImage + ParentProcessName: unmapped.ParentProcessName + PasswordLastSet: unmapped.PasswordLastSet + path: unmapped.path + PHYType: unmapped.PHYType + PipeName: unmapped.PipeName + PossibleCause: unmapped.PossibleCause + post-body: unmapped.post-body + PreAuthType: unmapped.PreAuthType + PreviousCreationUtcTime: unmapped.PreviousCreationUtcTime + PrivilegeList: unmapped.PrivilegeList + Process_Name: unmapped.Process_Name + ProcessName: unmapped.ProcessName + processPath: unmapped.processPath + ProfileName: unmapped.ProfileName + Properties: unmapped.Properties + properties.message: unmapped.properties.message + proto: unmapped.proto + protoPayload.methodName: unmapped.protoPayload.methodName + protoPayload.request.canIpForward: unmapped.protoPayload.request.canIpForward + protoPayload.serviceName: unmapped.protoPayload.serviceName + Provider: unmapped.Provider + Provider_Name: unmapped.Provider_Name + proxied: unmapped.proxied + QNAME: unmapped.QNAME + qtype_name: unmapped.qtype_name + query: unmapped.query + QueryName: unmapped.QueryName + QueryResults: unmapped.QueryResults + r-dns: unmapped.r-dns + rcode_name: unmapped.rcode_name + RecordType: unmapped.RecordType + referrer: unmapped.referrer + RegistryValue: unmapped.RegistryValue + rejected: unmapped.rejected + related.user: unmapped.related.user + RelativeTargetName: unmapped.RelativeTargetName + RemoteName: unmapped.RemoteName + request_body_len: unmapped.request_body_len + request_type: unmapped.request_type + requestObject.rules.resources: unmapped.requestObject.rules.resources + requestObject.rules.verbs: unmapped.requestObject.rules.verbs + requestObject.spec.containers.image: unmapped.requestObject.spec.containers.image + requestParameters: unmapped.requestParameters + requestParameters.attribute: unmapped.requestParameters.attribute + requestParameters.ipPermissions.items.ipRanges.items.cidrIP: unmapped.requestParameters.ipPermissions.items.ipRanges.items.cidrIP + requestParameters.ipPermissions.items.ipRanges.items.fromPort: unmapped.requestParameters.ipPermissions.items.ipRanges.items.fromPort + requestParameters.userData: unmapped.requestParameters.userData + requestURI: unmapped.requestURI + ResourceDisplayName: unmapped.ResourceDisplayName + ResourceId: unmapped.ResourceId + ResourceIdentity: unmapped.ResourceIdentity + ResourceProviderValue: unmapped.ResourceProviderValue + resp_bytes: unmapped.resp_bytes + resp_mime_types: unmapped.resp_mime_types + response_body_len: unmapped.response_body_len + responseElements.pendingModifiedValues.masterUserPassword: unmapped.responseElements.pendingModifiedValues.masterUserPassword + responseElements.publiclyAccessible: unmapped.responseElements.publiclyAccessible + Result: unmapped.Result + ResultDescription: unmapped.ResultDescription + ResultReason: unmapped.ResultReason + ResultStatus: unmapped.ResultStatus + ResultType: unmapped.ResultType + RiskDetail: unmapped.RiskDetail + sc-status: unmapped.sc-status + SearchFilter: unmapped.SearchFilter + SecurityDescriptor: unmapped.SecurityDescriptor + SecurityID: unmapped.SecurityID + SecurityPackageName: unmapped.SecurityPackageName + server_name: unmapped.server_name + ServerName: unmapped.ServerName + service: unmapped.service + ServiceAccount: unmapped.ServiceAccount + ServiceFileName: unmapped.ServiceFileName + serviceName: unmapped.serviceName + ServicePrincipalNames: unmapped.ServicePrincipalNames + ServiceSid: unmapped.ServiceSid + ServiceStartType: unmapped.ServiceStartType + ServiceType: unmapped.ServiceType + SessionName: unmapped.SessionName + share_name: unmapped.share_name + ShareName: unmapped.ShareName + SidHistory: unmapped.SidHistory + SignatureStatus: unmapped.SignatureStatus + Signed: unmapped.Signed + source: unmapped.source + Source_Name: unmapped.Source_Name + source.domain: unmapped.source.domain + source.ip: unmapped.source.ip + source.user.full_name: unmapped.source.user.full_name + source.user.id: unmapped.source.user.id + SourceAddr: unmapped.SourceAddr + SourceAddress: unmapped.SourceAddress + SourceFileExtension: unmapped.SourceFileExtension + SourceFileName: unmapped.SourceFileName + SourceImage: unmapped.SourceImage + SourceIp: unmapped.SourceIp + SourceItemName: unmapped.SourceItemName + SourceName: unmapped.SourceName + SourcePort: unmapped.SourcePort + SourceSystem: unmapped.SourceSystem + SourceUser: unmapped.SourceUser + src_ip: unmapped.src_ip + src_port: unmapped.src_port + src-ip: unmapped.src-ip + src-port: unmapped.src-port + SSID: unmapped.SSID + stage: unmapped.stage + StartType: unmapped.StartType + statement: unmapped.statement + status: unmapped.status + status_code: unmapped.status_code + Status.errorCode: unmapped.Status.errorCode + Status.failureReason: unmapped.Status.failureReason + StatusText: unmapped.StatusText + subject: unmapped.subject + SubjectDomainName: unmapped.SubjectDomainName + SubjectLogonId: unmapped.SubjectLogonId + SubjectUserName: unmapped.SubjectUserName + SubjectUserSid: unmapped.SubjectUserSid + SubStatus: unmapped.SubStatus + subsystem: unmapped.subsystem + successful: unmapped.successful + TargetDomainName: unmapped.TargetDomainName + TargetImage: unmapped.TargetImage + TargetInfo: unmapped.TargetInfo + TargetLogonId: unmapped.TargetLogonId + TargetName: unmapped.TargetName + TargetResources: unmapped.TargetResources + TargetServerName: unmapped.TargetServerName + TargetSid: unmapped.TargetSid + TargetUser: unmapped.TargetUser + TargetUserName: unmapped.TargetUserName + TargetUserSid: unmapped.TargetUserSid + TaskContent: unmapped.TaskContent + TaskName: unmapped.TaskName + TemplateContent: unmapped.TemplateContent + terminatingRuleId: unmapped.terminatingRuleId + TestSigning: unmapped.TestSigning + TicketEncryptionType: unmapped.TicketEncryptionType + TicketOptions: unmapped.TicketOptions + till: unmapped.till + TokenIssuerType: unmapped.TokenIssuerType + total_bytes: unmapped.total_bytes + TransmittedServices: unmapped.TransmittedServices + Type: unmapped.Type + uri: unmapped.uri + User: unmapped.User + user_agent: unmapped.user_agent + user_agent.original: unmapped.user_agent.original + user.groups: unmapped.user.groups + user.username: unmapped.user.username + UserAccountControl: unmapped.UserAccountControl + UserAgent: unmapped.UserAgent + userAgent.browser: unmapped.userAgent.browser + userAgent.os: unmapped.userAgent.os + UserContext: unmapped.UserContext + UserID: unmapped.UserID + UserName: unmapped.UserName + UserPrincipalName: unmapped.UserPrincipalName + UsersInsights: unmapped.UsersInsights + UsersInsights.IsDormantAccount: unmapped.UsersInsights.IsDormantAccount + UsersInsights.IsNewAccount: unmapped.UsersInsights.IsNewAccount + validation_status: unmapped.validation_status + Value: unmapped.Value + verb: unmapped.verb + Workstation: unmapped.Workstation + WorkstationName: unmapped.WorkstationName diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/default.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/default.yml new file mode 100644 index 00000000..6ae2dc90 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/default.yml @@ -0,0 +1,7 @@ +platform: Microsoft Sentinel +source: default + + + +default_log_source: + table: union * \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/proxy.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/proxy.yml new file mode 100644 index 00000000..7c22de40 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/proxy.yml @@ -0,0 +1,11 @@ +platform: Microsoft Sentinel +source: proxy + +default_log_source: + table: union * + +field_mapping: + dst_ip: dst_ip_addr + dst_port: dst_port + src_ip: src_ip_addr + src_port: src_port \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/webserver.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/webserver.yml new file mode 100644 index 00000000..4367f760 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/webserver.yml @@ -0,0 +1,11 @@ +platform: Microsoft Sentinel +source: webserver + +default_log_source: + table: union * + +field_mapping: + dst_ip: dst_ip_addr + dst_port: dst_port + src_ip: src_ip_addr + src_port: src_port diff --git a/uncoder-core/app/translator/mappings/utils/load_from_files.py b/uncoder-core/app/translator/mappings/utils/load_from_files.py index 89ea2a94..30684f17 100644 --- a/uncoder-core/app/translator/mappings/utils/load_from_files.py +++ b/uncoder-core/app/translator/mappings/utils/load_from_files.py @@ -7,6 +7,7 @@ COMMON_FIELD_MAPPING_FILE_NAME = "common.yml" DEFAULT_FIELD_MAPPING_FILE_NAME = "default.yml" +ALTERNATIVE_MAPPINGS_FOLDER_NAME = "alternative" class LoaderFileMappings: @@ -21,10 +22,22 @@ def load_mapping(mapping_file_path: str) -> dict: print(err) return {} + def get_platform_alternative_mappings_dirs(self, platform_dir: str) -> dict[str:str]: + platform_path = os.path.join(self.base_mapping_filepath, platform_dir, ALTERNATIVE_MAPPINGS_FOLDER_NAME) + for _, dirs, _ in os.walk(platform_path): + result = {} + for folder in dirs: + result[folder] = os.path.join(platform_dir, ALTERNATIVE_MAPPINGS_FOLDER_NAME, folder) + return result + return {} + def load_platform_mappings(self, platform_dir: str) -> Generator[dict, None, None]: platform_path = os.path.join(self.base_mapping_filepath, platform_dir) for mapping_file in os.listdir(platform_path): - if mapping_file not in (COMMON_FIELD_MAPPING_FILE_NAME, DEFAULT_FIELD_MAPPING_FILE_NAME): + if mapping_file.endswith(".yml") and mapping_file not in ( + COMMON_FIELD_MAPPING_FILE_NAME, + DEFAULT_FIELD_MAPPING_FILE_NAME, + ): yield self.load_mapping(mapping_file_path=os.path.join(platform_path, mapping_file)) yield self.load_mapping(mapping_file_path=os.path.join(platform_path, DEFAULT_FIELD_MAPPING_FILE_NAME)) diff --git a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py index 22b135cc..6ab5ce97 100644 --- a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py +++ b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py @@ -1,15 +1,15 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render_cti import RenderCTI from app.translator.managers import render_cti_manager -from app.translator.platforms.arcsight.const import arcsight_query_details, DEFAULT_ARCSIGHT_CTI_MAPPING +from app.translator.platforms.arcsight.const import DEFAULT_ARCSIGHT_CTI_MAPPING, arcsight_query_details @render_cti_manager.register class ArcsightKeyword(RenderCTI): details: PlatformDetails = arcsight_query_details + field_value_template: str = '{key} = "{value}"' default_mapping = DEFAULT_ARCSIGHT_CTI_MAPPING - field_value_template: str = "{key} = {value}" or_operator: str = " OR " group_or_operator: str = " OR " or_group: str = "{or_group}" diff --git a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py index 0dad8283..0881459d 100644 --- a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py @@ -17,7 +17,7 @@ """ import re -from typing import Union +from typing import Optional, Union from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.models.functions.base import ParsedFunctions @@ -105,8 +105,8 @@ def __parse_log_sources(self, query: str) -> tuple[dict[str, Union[list[str], li return log_sources, query - def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[int]]], ParsedFunctions]: - query = self.__clean_query(text) + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + query = self.__clean_query(query) self.__check_table(query) query, functions = self.platform_functions.parse(query) log_sources, query = self.__parse_log_sources(query) @@ -118,7 +118,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( query_tokens, functions.functions ) - source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens + function_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.function_fields = function_field_tokens diff --git a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py index 49f05c98..3a235ff9 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py @@ -17,7 +17,9 @@ """ import re +from typing import Optional, Union +from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.lucene.tokenizer import LuceneTokenizer @@ -31,7 +33,7 @@ class LuceneQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: log_sources = {} for source_type in self.log_source_key_types: pattern = self.log_source_pattern.replace("___source_type___", source_type) @@ -43,13 +45,17 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: pos_end = search.end() query = query[:pos_start] + query[pos_end:] - return query, log_sources + return query, log_sources, None def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - query, log_sources = self._parse_query(raw_query_container.query) + query, log_sources, _ = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(query_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] diff --git a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py index f56af913..fe62c227 100644 --- a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py @@ -75,7 +75,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( query_tokens, functions.functions ) - source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens + function_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.function_fields = function_field_tokens diff --git a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py index 01be3500..1fd62058 100644 --- a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py @@ -17,7 +17,9 @@ """ import re +from typing import Optional, Union +from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.sql.tokenizer import SqlTokenizer @@ -30,21 +32,25 @@ class SqlQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*--.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: log_source = {"table": []} if re.search(self.query_delimiter_pattern, query, flags=re.IGNORECASE): table_search = re.search(self.table_pattern, query) table = table_search.group("table") log_source["table"] = [table] - return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source + return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source, None - return query, log_source + return query, log_source, None def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - query, log_sources = self._parse_query(raw_query_container.query) + query, log_sources, _ = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(query_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py index 0cc1af82..e0af8a22 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py @@ -36,7 +36,11 @@ class ChronicleQueryParser(PlatformQueryParser): def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query_tokens = self.get_query_tokens(raw_query_container.query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(query_field_tokens, {}) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens, + log_sources={}, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py index 377b1e08..95c4e0f4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py @@ -30,7 +30,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query, log_sources = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(query_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py b/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py index dc70f44e..f57961a5 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py @@ -18,9 +18,9 @@ def __str__(self) -> str: class LogRhythmAxonMappings(BasePlatformMappings): is_strict_mapping = True - def prepare_mapping(self) -> dict[str, SourceMapping]: + def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: source_mappings = {} - for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): + for mapping_dict in self._loader.load_platform_mappings(platform_dir): log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) fields_mapping = self.prepare_fields_mapping(field_mapping=mapping_dict.get("field_mapping", {})) source_mappings[DEFAULT_MAPPING_NAME] = SourceMapping( diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py index ddf2fcd1..fbb949ce 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py @@ -46,7 +46,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( query_tokens, functions.functions ) - source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, {}) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens + function_field_tokens, + log_sources={}, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.function_fields = function_field_tokens diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index e7392ea3..0d4a9b57 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -16,6 +16,7 @@ ----------------------------------------------------------------- """ +from typing import Optional, Union from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.platform_details import PlatformDetails @@ -37,7 +38,7 @@ class MicrosoftSentinelQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFunctions]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: table, query, functions = self.platform_functions.parse(query) log_sources = {"table": [table]} return query, log_sources, functions @@ -48,7 +49,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( query_tokens, functions.functions ) - source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens + function_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.function_fields = function_field_tokens diff --git a/uncoder-core/app/translator/platforms/splunk/mapping.py b/uncoder-core/app/translator/platforms/splunk/mapping.py index b5750532..cff045ec 100644 --- a/uncoder-core/app/translator/platforms/splunk/mapping.py +++ b/uncoder-core/app/translator/platforms/splunk/mapping.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import ClassVar, Optional from app.translator.core.mapping import BasePlatformMappings, LogSourceSignature from app.translator.platforms.splunk.const import splunk_alert_details, splunk_query_details @@ -39,9 +39,13 @@ def __str__(self) -> str: class SplunkMappings(BasePlatformMappings): + global_alternative_mappings: ClassVar[list[str]] = ["ocsf"] + def prepare_log_source_signature(self, mapping: dict) -> SplunkLogSourceSignature: log_source = mapping.get("log_source", {}) - default_log_source = mapping["default_log_source"] + default_log_source = ( + mapping.get("default_log_source") if mapping.get("default_log_source") else {"source": "WinEventLog: *"} + ) return SplunkLogSourceSignature( sources=log_source.get("source"), source_types=log_source.get("sourcetype"), diff --git a/uncoder-core/app/translator/translator.py b/uncoder-core/app/translator/translator.py index 746ad3bb..80a35b58 100644 --- a/uncoder-core/app/translator/translator.py +++ b/uncoder-core/app/translator/translator.py @@ -68,12 +68,19 @@ def parse_meta_info(self, text: str, source: str) -> Union[dict, RawQueryContain @handle_translation_exceptions def __parse_incoming_data( - self, text: str, source: str, target: Optional[str] = None + self, + text: str, + source: str, + target: Optional[str] = None, + source_alt_mapping: Optional[str] = None, + target_alt_mapping: Optional[str] = None, ) -> tuple[RawQueryContainer, Optional[TokenizedQueryContainer]]: parser, raw_query_container = self.parse_raw_query(text=text, source=source) + raw_query_container.meta_info.source_alt_mapping = source_alt_mapping tokenized_query_container = None if not (target and self.__is_one_vendor_translation(raw_query_container.language, target)): tokenized_query_container = parser.parse(raw_query_container) + tokenized_query_container.meta_info.target_alt_mapping = target_alt_mapping return raw_query_container, tokenized_query_container @@ -89,8 +96,21 @@ def __render_translation( raw_query_container=raw_query_container, tokenized_query_container=tokenized_query_container ) - def __translate_one(self, text: str, source: str, target: str) -> (bool, str): - status, parsed_data = self.__parse_incoming_data(text=text, source=source, target=target) + def __translate_one( + self, + text: str, + source: str, + target: str, + source_alt_mapping: Optional[str] = None, + target_alt_mapping: Optional[str] = None, + ) -> (bool, str): + status, parsed_data = self.__parse_incoming_data( + text=text, + source=source, + target=target, + source_alt_mapping=source_alt_mapping, + target_alt_mapping=target_alt_mapping, + ) if not status: return status, parsed_data @@ -124,10 +144,23 @@ def __translate_all(self, text: str, source: str) -> list[dict]: return result - def translate_one(self, text: str, source: str, target: str) -> (bool, str): - if source == target: + def translate_one( + self, + text: str, + source: str, + target: str, + source_alt_mapping: Optional[str] = None, + target_alt_mapping: Optional[str] = None, + ) -> (bool, str): + if source == target and not (target_alt_mapping or source_alt_mapping): return True, text - return self.__translate_one(text=text, source=source, target=target) + return self.__translate_one( + text=text, + source=source, + target=target, + source_alt_mapping=source_alt_mapping, + target_alt_mapping=target_alt_mapping, + ) def translate_all(self, text: str, source: str) -> list[dict]: return self.__translate_all(text=text, source=source) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy