From cd8749f0974a79292c2a227e836b3b93b2e8e05d Mon Sep 17 00:00:00 2001 From: rm-socprime <86658859+rm-socprime@users.noreply.github.com> Date: Mon, 6 Jan 2025 12:21:39 +0100 Subject: [PATCH 01/15] Merge branch 'gis-add-anomali-mappings1612' into 'prod' windows mappings added See merge request tdm_backends/uncoder-group/uncoder-core!408 From 10a7c31231501de087af3739e338b3977e0aa6cb Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:22:37 +0200 Subject: [PATCH 02/15] Merge branch 'gis-9241' into 'prod' gis-9241 fix ArcsightKeyword cti See merge request tdm_backends/uncoder-group/uncoder-core!409 --- .../app/translator/platforms/arcsight/renders/arcsight_cti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py index 22b135cc..63fe8f90 100644 --- a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py +++ b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py @@ -8,8 +8,8 @@ class ArcsightKeyword(RenderCTI): details: PlatformDetails = arcsight_query_details + field_value_template: str = '{key} = "{value}"' default_mapping = DEFAULT_ARCSIGHT_CTI_MAPPING - field_value_template: str = "{key} = {value}" or_operator: str = " OR " group_or_operator: str = " OR " or_group: str = "{or_group}" From e4602e017868f8962375b87bca88ace30204e538 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:35:59 +0200 Subject: [PATCH 03/15] gis-9415 Add alternative translations functionality --- uncoder-core/app/routers/translate.py | 20 ++++++---- uncoder-core/app/translator/core/mapping.py | 38 +++++++++++++++---- .../app/translator/core/mixins/tokens.py | 2 - .../translator/core/models/query_container.py | 4 ++ uncoder-core/app/translator/core/render.py | 10 ++++- .../alternative/cloudtrail/cloudtrail.yml | 13 +++++++ .../alternative/cloudtrail/default.yml | 7 ++++ .../alternative/ossem/default.yml | 7 ++++ .../mappings/utils/load_from_files.py | 15 +++++++- .../arcsight/renders/arcsight_cti.py | 2 +- .../platforms/logrhythm_axon/mapping.py | 4 +- uncoder-core/app/translator/translator.py | 35 ++++++++++++++--- 12 files changed, 131 insertions(+), 26 deletions(-) create mode 100644 uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/cloudtrail.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/default.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/default.yml diff --git a/uncoder-core/app/routers/translate.py b/uncoder-core/app/routers/translate.py index 009cab03..b1f3867d 100644 --- a/uncoder-core/app/routers/translate.py +++ b/uncoder-core/app/routers/translate.py @@ -3,23 +3,29 @@ from app.models.translation import InfoMessage, OneTranslationData, Platform, TranslatorPlatforms from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.cti_translator import CTITranslator -from app.translator.translator import Translator +from app.translator.translator import app_translator st_router = APIRouter() -translator = Translator() - @st_router.post("/translate", tags=["translator"], description="Generate target translation") @st_router.post("/translate/", include_in_schema=False) def translate_one( source_platform_id: str = Body(..., embed=True), + source_alt_mapping: dict = Body(None, embed=True), target_platform_id: str = Body(..., embed=True), + target_alt_mapping: dict = Body(None, embed=True), text: str = Body(..., embed=True), return_only_first_query: bool = False, ) -> OneTranslationData: return_only_first_query_ctx_var.set(return_only_first_query) - status, data = translator.translate_one(text=text, source=source_platform_id, target=target_platform_id) + status, data = app_translator.translate_one( + text=text, + source=source_platform_id, + target=target_platform_id, + source_alt_mapping=source_alt_mapping, + target_alt_mapping=target_alt_mapping, + ) if status: return OneTranslationData(status=status, translation=data, target_platform_id=target_platform_id) @@ -35,7 +41,7 @@ def translate_all( return_only_first_query: bool = False, ) -> list[OneTranslationData]: return_only_first_query_ctx_var.set(return_only_first_query) - result = translator.translate_all(text=text, source=source_platform_id) + result = app_translator.translate_all(text=text, source=source_platform_id) translations = [] for platform_result in result: if platform_result.get("status"): @@ -60,14 +66,14 @@ def translate_all( @st_router.get("/platforms", tags=["translator"], description="Get translator platforms") @st_router.get("/platforms/", include_in_schema=False) def get_translator_platforms() -> TranslatorPlatforms: - renders, parsers = translator.get_all_platforms() + renders, parsers = app_translator.get_all_platforms() return TranslatorPlatforms(renders=renders, parsers=parsers) @st_router.get("/all_platforms", description="Get Sigma, RootA and iocs platforms") @st_router.get("/all_platforms/", include_in_schema=False) def get_all_platforms() -> list: - translator_renders, translator_parsers = translator.get_all_platforms() + translator_renders, translator_parsers = app_translator.get_all_platforms() return [ Platform( id="roota", diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 1c4d2070..1a3d917b 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -112,17 +112,23 @@ class BasePlatformMappings: def __init__(self, platform_dir: str, platform_details: PlatformDetails): self._loader = LoaderFileMappings() - self._platform_dir = platform_dir self.details = platform_details - self._source_mappings = self.prepare_mapping() + self._source_mappings = self.prepare_mapping(platform_dir) + self._alternative_mappings = self.prepare_alternative_mapping(platform_dir) def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None: default_mapping.fields_mapping.update(fields_mapping) - def prepare_mapping(self) -> dict[str, SourceMapping]: + def prepare_alternative_mapping(self, platform_dir: str) -> dict[str, dict[str, SourceMapping]]: + alternative_mappings = {} + for name, platform_dir in self._loader.get_platform_alternative_mappings(platform_dir).items(): + alternative_mappings[name] = self.prepare_mapping(platform_dir) + return alternative_mappings + + def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: source_mappings = {} default_mapping = SourceMapping(source_id=DEFAULT_MAPPING_NAME) - for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): + for mapping_dict in self._loader.load_platform_mappings(platform_dir): log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) if (source_id := mapping_dict["source"]) == DEFAULT_MAPPING_NAME: default_mapping.log_source_signature = log_source_signature @@ -183,6 +189,9 @@ def get_source_mappings_by_fields_and_log_sources( def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]: return self._source_mappings.get(source_id) + def get_alternative_source_mapping(self, alt_config_name: str, source_id: str) -> Optional[SourceMapping]: + return self._alternative_mappings.get(alt_config_name, {}).get(source_id) + def get_source_mappings_by_ids( self, source_mapping_ids: list[str], return_default: bool = True ) -> list[SourceMapping]: @@ -198,6 +207,21 @@ def get_source_mappings_by_ids( return source_mappings + def get_alternative_source_mappings_by_ids( + self, source_mapping_ids: list[str], alt_config_name: str, return_default: bool = True + ) -> list[SourceMapping]: + source_mappings = [] + for source_mapping_id in source_mapping_ids: + if source_mapping_id == DEFAULT_MAPPING_NAME: + continue + if source_mapping := self.get_alternative_source_mapping(alt_config_name, source_mapping_id): + source_mappings.append(source_mapping) + + if not source_mappings and return_default: + source_mappings = [self.get_alternative_source_mapping(alt_config_name, DEFAULT_MAPPING_NAME)] + + return source_mappings + def get_source_mappings_by_log_sources(self, log_sources: dict) -> Optional[list[str]]: raise NotImplementedError("Abstract method") @@ -249,11 +273,11 @@ def map_field(field: Field, source_mapping: SourceMapping) -> list[str]: class BaseCommonPlatformMappings(ABC, BasePlatformMappings): - def prepare_mapping(self) -> dict[str, SourceMapping]: + def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: source_mappings = {} - common_field_mapping = self._loader.load_common_mapping(self._platform_dir).get("field_mapping", {}) + common_field_mapping = self._loader.load_common_mapping(platform_dir).get("field_mapping", {}) - for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): + for mapping_dict in self._loader.load_platform_mappings(platform_dir): source_id = mapping_dict["source"] log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) fields_mapping = self.prepare_fields_mapping(field_mapping=common_field_mapping) diff --git a/uncoder-core/app/translator/core/mixins/tokens.py b/uncoder-core/app/translator/core/mixins/tokens.py index 09bbe266..2bba49c7 100644 --- a/uncoder-core/app/translator/core/mixins/tokens.py +++ b/uncoder-core/app/translator/core/mixins/tokens.py @@ -1,5 +1,3 @@ -from typing import Union - from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.mapping import SourceMapping diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index ad866b51..c35804b2 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -75,6 +75,8 @@ def __init__( status: Optional[str] = None, false_positives: Optional[list[str]] = None, source_mapping_ids: Optional[list[str]] = None, + source_alt_mapping: Optional[str] = None, + target_alt_mapping: Optional[str] = None, parsed_logsources: Optional[dict] = None, timeframe: Optional[timedelta] = None, query_period: Optional[timedelta] = None, @@ -107,6 +109,8 @@ def __init__( self.timeframe = timeframe self.query_period = query_period self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer() + self.source_alt_mapping = source_alt_mapping + self.target_alt_mapping = target_alt_mapping @property def author_str(self) -> str: diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 673cc6fa..32cab632 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -463,7 +463,15 @@ def _generate_from_tokenized_query_container_by_source_mapping( def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} errors = [] - source_mappings = self.mappings.get_source_mappings_by_ids(query_container.meta_info.source_mapping_ids) + if query_container.meta_info.target_alt_mapping: + source_mappings = self.mappings.get_alternative_source_mappings_by_ids( + source_mapping_ids=query_container.meta_info.source_mapping_ids, + alt_config_name=query_container.meta_info.target_alt_mapping, + ) + else: + source_mappings = self.mappings.get_source_mappings_by_ids( + source_mapping_ids=query_container.meta_info.source_mapping_ids + ) for source_mapping in source_mappings: try: diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/cloudtrail.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/cloudtrail.yml new file mode 100644 index 00000000..51217cc8 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/cloudtrail.yml @@ -0,0 +1,13 @@ +platform: Microsoft Sentinel +source: cloudtrail + + +log_source: + table: [AWSCloudTrail] + +default_log_source: + table: AWSCloudTrail + +field_mapping: + eventSource: EventSource + eventName: EventName diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/default.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/default.yml new file mode 100644 index 00000000..6ae2dc90 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/default.yml @@ -0,0 +1,7 @@ +platform: Microsoft Sentinel +source: default + + + +default_log_source: + table: union * \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/default.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/default.yml new file mode 100644 index 00000000..6ae2dc90 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/default.yml @@ -0,0 +1,7 @@ +platform: Microsoft Sentinel +source: default + + + +default_log_source: + table: union * \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/utils/load_from_files.py b/uncoder-core/app/translator/mappings/utils/load_from_files.py index 89ea2a94..b60263a9 100644 --- a/uncoder-core/app/translator/mappings/utils/load_from_files.py +++ b/uncoder-core/app/translator/mappings/utils/load_from_files.py @@ -7,6 +7,7 @@ COMMON_FIELD_MAPPING_FILE_NAME = "common.yml" DEFAULT_FIELD_MAPPING_FILE_NAME = "default.yml" +DEFAULT_ALTERNATIVE_MAPPINGS_FOLDER_NAME = "alternative" class LoaderFileMappings: @@ -21,10 +22,22 @@ def load_mapping(mapping_file_path: str) -> dict: print(err) return {} + def get_platform_alternative_mappings(self, platform_dir: str) -> dict[str:str]: + platform_path = os.path.join(self.base_mapping_filepath, platform_dir, DEFAULT_ALTERNATIVE_MAPPINGS_FOLDER_NAME) + for folders in os.walk(platform_path): + result = {} + for folder in folders[1]: + result[folder] = os.path.join(platform_dir, DEFAULT_ALTERNATIVE_MAPPINGS_FOLDER_NAME, folder) + return result + return {} + def load_platform_mappings(self, platform_dir: str) -> Generator[dict, None, None]: platform_path = os.path.join(self.base_mapping_filepath, platform_dir) for mapping_file in os.listdir(platform_path): - if mapping_file not in (COMMON_FIELD_MAPPING_FILE_NAME, DEFAULT_FIELD_MAPPING_FILE_NAME): + if mapping_file.endswith(".yml") and mapping_file not in ( + COMMON_FIELD_MAPPING_FILE_NAME, + DEFAULT_FIELD_MAPPING_FILE_NAME, + ): yield self.load_mapping(mapping_file_path=os.path.join(platform_path, mapping_file)) yield self.load_mapping(mapping_file_path=os.path.join(platform_path, DEFAULT_FIELD_MAPPING_FILE_NAME)) diff --git a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py index 63fe8f90..6ab5ce97 100644 --- a/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py +++ b/uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py @@ -1,7 +1,7 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render_cti import RenderCTI from app.translator.managers import render_cti_manager -from app.translator.platforms.arcsight.const import arcsight_query_details, DEFAULT_ARCSIGHT_CTI_MAPPING +from app.translator.platforms.arcsight.const import DEFAULT_ARCSIGHT_CTI_MAPPING, arcsight_query_details @render_cti_manager.register diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py b/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py index dc70f44e..f57961a5 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py @@ -18,9 +18,9 @@ def __str__(self) -> str: class LogRhythmAxonMappings(BasePlatformMappings): is_strict_mapping = True - def prepare_mapping(self) -> dict[str, SourceMapping]: + def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: source_mappings = {} - for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): + for mapping_dict in self._loader.load_platform_mappings(platform_dir): log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) fields_mapping = self.prepare_fields_mapping(field_mapping=mapping_dict.get("field_mapping", {})) source_mappings[DEFAULT_MAPPING_NAME] = SourceMapping( diff --git a/uncoder-core/app/translator/translator.py b/uncoder-core/app/translator/translator.py index 746ad3bb..9c1c91e4 100644 --- a/uncoder-core/app/translator/translator.py +++ b/uncoder-core/app/translator/translator.py @@ -68,12 +68,19 @@ def parse_meta_info(self, text: str, source: str) -> Union[dict, RawQueryContain @handle_translation_exceptions def __parse_incoming_data( - self, text: str, source: str, target: Optional[str] = None + self, + text: str, + source: str, + target: Optional[str] = None, + source_alt_mapping: Optional[str] = None, + target_alt_mapping: Optional[str] = None, ) -> tuple[RawQueryContainer, Optional[TokenizedQueryContainer]]: parser, raw_query_container = self.parse_raw_query(text=text, source=source) + raw_query_container.meta_info.source_alt_mapping = source_alt_mapping tokenized_query_container = None if not (target and self.__is_one_vendor_translation(raw_query_container.language, target)): tokenized_query_container = parser.parse(raw_query_container) + tokenized_query_container.meta_info.target_alt_mapping = target_alt_mapping return raw_query_container, tokenized_query_container @@ -89,8 +96,17 @@ def __render_translation( raw_query_container=raw_query_container, tokenized_query_container=tokenized_query_container ) - def __translate_one(self, text: str, source: str, target: str) -> (bool, str): - status, parsed_data = self.__parse_incoming_data(text=text, source=source, target=target) + def __translate_one( + self, + text: str, + source: str, + target: str, + source_alt_mapping: Optional[str] = None, + target_alt_mapping: Optional[str] = None, + ) -> (bool, str): + status, parsed_data = self.__parse_incoming_data( + text=text, source=source, target=target, source_alt_mapping=source_alt_mapping, target_alt_mapping=target_alt_mapping + ) if not status: return status, parsed_data @@ -124,10 +140,19 @@ def __translate_all(self, text: str, source: str) -> list[dict]: return result - def translate_one(self, text: str, source: str, target: str) -> (bool, str): + def translate_one( + self, + text: str, + source: str, + target: str, + source_alt_mapping: Optional[str] = None, + target_alt_mapping: Optional[str] = None, + ) -> (bool, str): if source == target: return True, text - return self.__translate_one(text=text, source=source, target=target) + return self.__translate_one( + text=text, source=source, target=target, source_alt_mapping=source_alt_mapping, target_alt_mapping=target_alt_mapping + ) def translate_all(self, text: str, source: str) -> list[dict]: return self.__translate_all(text=text, source=source) From 66fcaa7304c70baa5604318d6445e46f1b369133 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Tue, 7 Jan 2025 10:56:57 +0200 Subject: [PATCH 04/15] add alt mapping to microsoft_sentinel parser --- uncoder-core/app/translator/core/mapping.py | 22 ++++++++++++++++ uncoder-core/app/translator/core/parser.py | 26 +++++++++++++++---- .../microsoft/parsers/microsoft_sentinel.py | 9 +++++-- uncoder-core/app/translator/translator.py | 18 +++++++++++-- 4 files changed, 66 insertions(+), 9 deletions(-) diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 1a3d917b..403d6985 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -186,6 +186,28 @@ def get_source_mappings_by_fields_and_log_sources( return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]] + def get_alt_source_mappings_by_fields_and_log_sources( + self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]], alt_mapping: str + ) -> list[SourceMapping]: + by_log_sources_and_fields = [] + by_fields = [] + for source_mapping in self._alternative_mappings.get(alt_mapping).values(): + if source_mapping.source_id == DEFAULT_MAPPING_NAME: + continue + + if source_mapping.fields_mapping.is_suitable(field_names): + by_fields.append(source_mapping) + + log_source_signature: LogSourceSignature = source_mapping.log_source_signature + if log_source_signature and log_source_signature.is_suitable(**log_sources): + by_log_sources_and_fields.append(source_mapping) + + return ( + by_log_sources_and_fields + or by_fields + or [self._alternative_mappings.get(alt_mapping)[DEFAULT_MAPPING_NAME]] + ) + def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]: return self._source_mappings.get(source_id) diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index da7330eb..69eae9b1 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -24,7 +24,7 @@ from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping -from app.translator.core.models.functions.base import Function +from app.translator.core.models.functions.base import Function, ParsedFunctions from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.models.query_tokens.field import Field @@ -51,6 +51,9 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: raise NotImplementedError("Abstract method") + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + raise NotImplementedError("Abstract method") + class PlatformQueryParser(QueryParser, ABC): mappings: BasePlatformMappings = None @@ -80,11 +83,24 @@ def get_field_tokens( return query_field_tokens, function_field_tokens, function_field_tokens_map def get_source_mappings( - self, field_tokens: list[Field], log_sources: dict[str, list[Union[int, str]]] + self, + field_tokens: list[Field], + log_sources: dict[str, list[Union[int, str]]], + alt_mapping: Optional[str] = None, ) -> list[SourceMapping]: field_names = [field.source_name for field in field_tokens] - source_mappings = self.mappings.get_source_mappings_by_fields_and_log_sources( - field_names=field_names, log_sources=log_sources - ) + if alt_mapping: + source_mappings = self.mappings.get_alt_source_mappings_by_fields_and_log_sources( + field_names=field_names, log_sources=log_sources, alt_mapping=alt_mapping + ) + else: + source_mappings = self.mappings.get_source_mappings_by_fields_and_log_sources( + field_names=field_names, log_sources=log_sources + ) self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping) return source_mappings + + def get_source_mapping_ids_by_logsources(self, query: str) -> Optional[list[str]]: + _, parsed_logsources, _ = self._parse_query(query=query) + if parsed_logsources: + return self.mappings.get_source_mappings_by_log_sources(parsed_logsources) diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index e7392ea3..eef311b3 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -16,6 +16,7 @@ ----------------------------------------------------------------- """ +from typing import Optional, Union from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.platform_details import PlatformDetails @@ -37,7 +38,7 @@ class MicrosoftSentinelQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFunctions]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: table, query, functions = self.platform_functions.parse(query) log_sources = {"table": [table]} return query, log_sources, functions @@ -48,7 +49,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( query_tokens, functions.functions ) - source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens = query_field_tokens + function_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.function_fields = function_field_tokens diff --git a/uncoder-core/app/translator/translator.py b/uncoder-core/app/translator/translator.py index 9c1c91e4..13231bd5 100644 --- a/uncoder-core/app/translator/translator.py +++ b/uncoder-core/app/translator/translator.py @@ -105,7 +105,11 @@ def __translate_one( target_alt_mapping: Optional[str] = None, ) -> (bool, str): status, parsed_data = self.__parse_incoming_data( - text=text, source=source, target=target, source_alt_mapping=source_alt_mapping, target_alt_mapping=target_alt_mapping + text=text, + source=source, + target=target, + source_alt_mapping=source_alt_mapping, + target_alt_mapping=target_alt_mapping, ) if not status: return status, parsed_data @@ -149,9 +153,19 @@ def translate_one( target_alt_mapping: Optional[str] = None, ) -> (bool, str): if source == target: + if target_alt_mapping or source_alt_mapping: + message = ( + "Currently, Uncoder doesn't support translation between " + "non-default data schemas of the same platform." + ) + return False, message return True, text return self.__translate_one( - text=text, source=source, target=target, source_alt_mapping=source_alt_mapping, target_alt_mapping=target_alt_mapping + text=text, + source=source, + target=target, + source_alt_mapping=source_alt_mapping, + target_alt_mapping=target_alt_mapping, ) def translate_all(self, text: str, source: str) -> list[dict]: From 68770af981305161040425e946880022c4128467 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:59:19 +0200 Subject: [PATCH 05/15] mapping fixes --- .../alternative/cloudtrail/cloudtrail.yml | 13 ------------- .../alternative/cloudtrail/default.yml | 7 ------- .../microsoft_sentinel/alternative/ossem/proxy.yml | 11 +++++++++++ .../alternative/ossem/webserver.yml | 11 +++++++++++ 4 files changed, 22 insertions(+), 20 deletions(-) delete mode 100644 uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/cloudtrail.yml delete mode 100644 uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/default.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/proxy.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/webserver.yml diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/cloudtrail.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/cloudtrail.yml deleted file mode 100644 index 51217cc8..00000000 --- a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/cloudtrail.yml +++ /dev/null @@ -1,13 +0,0 @@ -platform: Microsoft Sentinel -source: cloudtrail - - -log_source: - table: [AWSCloudTrail] - -default_log_source: - table: AWSCloudTrail - -field_mapping: - eventSource: EventSource - eventName: EventName diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/default.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/default.yml deleted file mode 100644 index 6ae2dc90..00000000 --- a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/default.yml +++ /dev/null @@ -1,7 +0,0 @@ -platform: Microsoft Sentinel -source: default - - - -default_log_source: - table: union * \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/proxy.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/proxy.yml new file mode 100644 index 00000000..7c22de40 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/proxy.yml @@ -0,0 +1,11 @@ +platform: Microsoft Sentinel +source: proxy + +default_log_source: + table: union * + +field_mapping: + dst_ip: dst_ip_addr + dst_port: dst_port + src_ip: src_ip_addr + src_port: src_port \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/webserver.yml b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/webserver.yml new file mode 100644 index 00000000..4367f760 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/webserver.yml @@ -0,0 +1,11 @@ +platform: Microsoft Sentinel +source: webserver + +default_log_source: + table: union * + +field_mapping: + dst_ip: dst_ip_addr + dst_port: dst_port + src_ip: src_ip_addr + src_port: src_port From 3a30dcf3bb8c2168bfabcdd313ed000e2970f6de Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:47:10 +0200 Subject: [PATCH 06/15] fixes --- uncoder-core/app/translator/core/mapping.py | 2 +- .../app/translator/mappings/utils/load_from_files.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 403d6985..ed01b27a 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -191,7 +191,7 @@ def get_alt_source_mappings_by_fields_and_log_sources( ) -> list[SourceMapping]: by_log_sources_and_fields = [] by_fields = [] - for source_mapping in self._alternative_mappings.get(alt_mapping).values(): + for source_mapping in self._alternative_mappings.get(alt_mapping, {}).values(): if source_mapping.source_id == DEFAULT_MAPPING_NAME: continue diff --git a/uncoder-core/app/translator/mappings/utils/load_from_files.py b/uncoder-core/app/translator/mappings/utils/load_from_files.py index b60263a9..f58c89cd 100644 --- a/uncoder-core/app/translator/mappings/utils/load_from_files.py +++ b/uncoder-core/app/translator/mappings/utils/load_from_files.py @@ -7,7 +7,7 @@ COMMON_FIELD_MAPPING_FILE_NAME = "common.yml" DEFAULT_FIELD_MAPPING_FILE_NAME = "default.yml" -DEFAULT_ALTERNATIVE_MAPPINGS_FOLDER_NAME = "alternative" +ALTERNATIVE_MAPPINGS_FOLDER_NAME = "alternative" class LoaderFileMappings: @@ -23,11 +23,11 @@ def load_mapping(mapping_file_path: str) -> dict: return {} def get_platform_alternative_mappings(self, platform_dir: str) -> dict[str:str]: - platform_path = os.path.join(self.base_mapping_filepath, platform_dir, DEFAULT_ALTERNATIVE_MAPPINGS_FOLDER_NAME) - for folders in os.walk(platform_path): + platform_path = os.path.join(self.base_mapping_filepath, platform_dir, ALTERNATIVE_MAPPINGS_FOLDER_NAME) + for _, dirs, _ in os.walk(platform_path): result = {} - for folder in folders[1]: - result[folder] = os.path.join(platform_dir, DEFAULT_ALTERNATIVE_MAPPINGS_FOLDER_NAME, folder) + for folder in dirs: + result[folder] = os.path.join(platform_dir, ALTERNATIVE_MAPPINGS_FOLDER_NAME, folder) return result return {} From ed36ada8013db4c8e0994b14da334e2f09793bfb Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 8 Jan 2025 12:03:21 +0200 Subject: [PATCH 07/15] fixes --- uncoder-core/app/translator/core/mapping.py | 33 ++++++++----------- .../mappings/utils/load_from_files.py | 2 +- .../microsoft/parsers/microsoft_sentinel.py | 4 +-- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index ed01b27a..a8606451 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -121,7 +121,7 @@ def update_default_source_mapping(self, default_mapping: SourceMapping, fields_m def prepare_alternative_mapping(self, platform_dir: str) -> dict[str, dict[str, SourceMapping]]: alternative_mappings = {} - for name, platform_dir in self._loader.get_platform_alternative_mappings(platform_dir).items(): + for name, platform_dir in self._loader.get_platform_alternative_mappings_dirs(platform_dir).items(): alternative_mappings[name] = self.prepare_mapping(platform_dir) return alternative_mappings @@ -170,10 +170,18 @@ def prepare_log_source_signature(self, mapping: dict) -> LogSourceSignature: def get_source_mappings_by_fields_and_log_sources( self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]] + ) -> list[SourceMapping]: + return self._get_source_mappings_by_fields_and_log_sources(field_names, log_sources, self._source_mappings) + + def _get_source_mappings_by_fields_and_log_sources( + self, + field_names: list[str], + log_sources: dict[str, list[Union[int, str]]], + source_mapping: dict[str, SourceMapping], ) -> list[SourceMapping]: by_log_sources_and_fields = [] by_fields = [] - for source_mapping in self._source_mappings.values(): + for source_mapping in source_mapping.values(): if source_mapping.source_id == DEFAULT_MAPPING_NAME: continue @@ -184,28 +192,13 @@ def get_source_mappings_by_fields_and_log_sources( if log_source_signature and log_source_signature.is_suitable(**log_sources): by_log_sources_and_fields.append(source_mapping) - return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]] + return by_log_sources_and_fields or by_fields or [source_mapping[DEFAULT_MAPPING_NAME]] def get_alt_source_mappings_by_fields_and_log_sources( self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]], alt_mapping: str ) -> list[SourceMapping]: - by_log_sources_and_fields = [] - by_fields = [] - for source_mapping in self._alternative_mappings.get(alt_mapping, {}).values(): - if source_mapping.source_id == DEFAULT_MAPPING_NAME: - continue - - if source_mapping.fields_mapping.is_suitable(field_names): - by_fields.append(source_mapping) - - log_source_signature: LogSourceSignature = source_mapping.log_source_signature - if log_source_signature and log_source_signature.is_suitable(**log_sources): - by_log_sources_and_fields.append(source_mapping) - - return ( - by_log_sources_and_fields - or by_fields - or [self._alternative_mappings.get(alt_mapping)[DEFAULT_MAPPING_NAME]] + return self._get_source_mappings_by_fields_and_log_sources( + field_names, log_sources, self._alternative_mappings.get(alt_mapping, {}) ) def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]: diff --git a/uncoder-core/app/translator/mappings/utils/load_from_files.py b/uncoder-core/app/translator/mappings/utils/load_from_files.py index f58c89cd..30684f17 100644 --- a/uncoder-core/app/translator/mappings/utils/load_from_files.py +++ b/uncoder-core/app/translator/mappings/utils/load_from_files.py @@ -22,7 +22,7 @@ def load_mapping(mapping_file_path: str) -> dict: print(err) return {} - def get_platform_alternative_mappings(self, platform_dir: str) -> dict[str:str]: + def get_platform_alternative_mappings_dirs(self, platform_dir: str) -> dict[str:str]: platform_path = os.path.join(self.base_mapping_filepath, platform_dir, ALTERNATIVE_MAPPINGS_FOLDER_NAME) for _, dirs, _ in os.walk(platform_path): result = {} diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index eef311b3..0d4a9b57 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -50,9 +50,9 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query_tokens, functions.functions ) source_mappings = self.get_source_mappings( - field_tokens = query_field_tokens + function_field_tokens, + field_tokens=query_field_tokens + function_field_tokens, log_sources=log_sources, - alt_mapping=raw_query_container.meta_info.source_alt_mapping + alt_mapping=raw_query_container.meta_info.source_alt_mapping, ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens From 9a9632f61fac298722b647cb705c1646befe0981 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:27:40 +0200 Subject: [PATCH 08/15] gis-9415 fix --- uncoder-core/app/translator/core/mapping.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index a8606451..5f9989d9 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -177,11 +177,11 @@ def _get_source_mappings_by_fields_and_log_sources( self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]], - source_mapping: dict[str, SourceMapping], + source_mappings: dict[str, SourceMapping], ) -> list[SourceMapping]: by_log_sources_and_fields = [] by_fields = [] - for source_mapping in source_mapping.values(): + for source_mapping in source_mappings.values(): if source_mapping.source_id == DEFAULT_MAPPING_NAME: continue @@ -192,7 +192,7 @@ def _get_source_mappings_by_fields_and_log_sources( if log_source_signature and log_source_signature.is_suitable(**log_sources): by_log_sources_and_fields.append(source_mapping) - return by_log_sources_and_fields or by_fields or [source_mapping[DEFAULT_MAPPING_NAME]] + return by_log_sources_and_fields or by_fields or [source_mappings[DEFAULT_MAPPING_NAME]] def get_alt_source_mappings_by_fields_and_log_sources( self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]], alt_mapping: str From 8e606cafc74dc5f22e055316e20f147fdb10ea67 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:35:38 +0200 Subject: [PATCH 09/15] gis-9415 add exception fro alternative mappings --- uncoder-core/app/translator/core/exceptions/core.py | 8 ++++++++ uncoder-core/app/translator/core/mapping.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/core/exceptions/core.py b/uncoder-core/app/translator/core/exceptions/core.py index 425c1ff0..3ded7f24 100644 --- a/uncoder-core/app/translator/core/exceptions/core.py +++ b/uncoder-core/app/translator/core/exceptions/core.py @@ -42,6 +42,14 @@ def __init__(self, platform: str, is_parser: bool = False): super().__init__(message) +class UnsupportedPlatformAlternativeMapping(BasePlatformException): + def __init__(self, platform: str, alt_mapping: str): + message = ( + f"The selected data schema '{alt_mapping}' doesn't have a field mapping for this log source in {platform}." + ) + super().__init__(message) + + class UnsupportedRootAParser(BasePlatformException): def __init__(self, parser: str): message = ( diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 5f9989d9..b9038314 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -3,7 +3,11 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Optional, TypeVar, Union -from app.translator.core.exceptions.core import StrictPlatformException, UnsupportedMappingsException +from app.translator.core.exceptions.core import ( + StrictPlatformException, + UnsupportedMappingsException, + UnsupportedPlatformAlternativeMapping, +) from app.translator.core.models.platform_details import PlatformDetails from app.translator.mappings.utils.load_from_files import LoaderFileMappings @@ -205,7 +209,9 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]: return self._source_mappings.get(source_id) def get_alternative_source_mapping(self, alt_config_name: str, source_id: str) -> Optional[SourceMapping]: - return self._alternative_mappings.get(alt_config_name, {}).get(source_id) + if self._alternative_mappings.get(alt_config_name): + return self._alternative_mappings.get(alt_config_name).get(source_id) + raise UnsupportedPlatformAlternativeMapping(platform=self.details.platform_id, alt_mapping=alt_config_name) def get_source_mappings_by_ids( self, source_mapping_ids: list[str], return_default: bool = True From 0383ff342f729d7aaa3a887eaf908999f3cafcac Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Mon, 20 Jan 2025 10:00:36 +0200 Subject: [PATCH 10/15] gis-9415 add global alternative mappings --- uncoder-core/app/translator/core/mapping.py | 6 + .../global_alternative/ocsf/default.yml | 434 ++++++++++++++++++ 2 files changed, 440 insertions(+) create mode 100644 uncoder-core/app/translator/mappings/global_alternative/ocsf/default.yml diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index b9038314..b99bee54 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -113,12 +113,14 @@ class BasePlatformMappings: is_strict_mapping: bool = False skip_load_default_mappings: bool = True extend_default_mapping_with_all_fields: bool = False + global_mappings: list[str] = [] def __init__(self, platform_dir: str, platform_details: PlatformDetails): self._loader = LoaderFileMappings() self.details = platform_details self._source_mappings = self.prepare_mapping(platform_dir) self._alternative_mappings = self.prepare_alternative_mapping(platform_dir) + global_alternative_mappings = self.prepare_global_alternative_mapping() def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None: default_mapping.fields_mapping.update(fields_mapping) @@ -129,6 +131,10 @@ def prepare_alternative_mapping(self, platform_dir: str) -> dict[str, dict[str, alternative_mappings[name] = self.prepare_mapping(platform_dir) return alternative_mappings + def prepare_global_alternative_mapping(self) -> dict[str, dict[str, SourceMapping]]: + globa_alternative_mappings = {} + return globa_alternative_mappings + def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: source_mappings = {} default_mapping = SourceMapping(source_id=DEFAULT_MAPPING_NAME) diff --git a/uncoder-core/app/translator/mappings/global_alternative/ocsf/default.yml b/uncoder-core/app/translator/mappings/global_alternative/ocsf/default.yml new file mode 100644 index 00000000..361135e3 --- /dev/null +++ b/uncoder-core/app/translator/mappings/global_alternative/ocsf/default.yml @@ -0,0 +1,434 @@ +platform: Global OCSF +source: default + +default_log_source: {} + +fieldmappings: + IntegrityLevel: process.integrity + ParentCommandLine: + - process.parent_process.cmd_line + - actor.process.cmd_line + ParentUser: process.parent_process.user + Product: + - process.file.product.name + - process.file.product.vendor_name + TargetFilename: file.name + OriginalFileName: module.file.path + TargetObject: + - reg_key.path + - reg_value.name + - reg_value.path + AdditionalEventData: unmapped.additionalEventData + additionalEventData.MFAUsed: unmapped.additionalEventData.MFAUsed + eventType: unmapped.eventType + responseElements: unmapped.responseElements + responseElements.ConsoleLogin: unmapped.responseElements.ConsoleLogin + userAgent: http_request.user_agent + userIdentity.arn: unmapped.userIdentity_sessionContext_sessionIssuer_arn + userIdentity.principalId: unmapped.userIdentity_sessionContext_sessionIssuer_principalId + userIdentity.sessionContext.sessionIssuer.type: unmapped.userIdentity_sessionContext_sessionIssuer_type + userIdentity.type: identity.user.type + userIdentity.userName: unmapped.userIdentity_sessionContext_sessionIssuer_userName + a0: unmapped.a0 + a1: unmapped.a1 + a2: unmapped.a2 + a3: unmapped.a3 + AADOperationType: unmapped.AADOperationType + Accesses: unmapped.Accesses + AccessMask: unmapped.AccessMask + AccountName: unmapped.AccountName + action: unmapped.action + action_id: unmapped.action_id + ActionType: unmapped.ActionType + ActivityDisplayName: unmapped.ActivityDisplayName + ActivityInsights: unmapped.ActivityInsights + ActivityStatus: unmapped.ActivityStatus + ActivityStatusValue: unmapped.ActivityStatusValue + ActivitySubstatusValue: unmapped.ActivitySubstatusValue + ActivityType: unmapped.ActivityType + additional_information: unmapped.additional_information + Address: unmapped.Address + AllowedToDelegateTo: unmapped.AllowedToDelegateTo + alternateId: unmapped.alternateId + answers: unmapped.answers + AppDisplayName: unmapped.AppDisplayName + AppId: unmapped.AppId + ApplicationPath: unmapped.ApplicationPath + attackName: unmapped.attackName + AttributeLDAPDisplayName: unmapped.AttributeLDAPDisplayName + AttributeValue: unmapped.AttributeValue + AuditPolicyChanges: unmapped.AuditPolicyChanges + AuditSourceName: unmapped.AuditSourceName + AuthenticationAlgorithm: unmapped.AuthenticationAlgorithm + AuthenticationPackageName: unmapped.AuthenticationPackageName + AuthenticationRequirement: unmapped.AuthenticationRequirement + AuthenticationType: unmapped.AuthenticationType + Authorization: unmapped.Authorization + aws_node_type: unmapped.aws_node_type + backupConfiguration.enabled: unmapped.backupConfiguration.enabled + BSSID: unmapped.BSSID + BSSType: unmapped.BSSType + c-uri: unmapped.c-uri + c-uri-extension: unmapped.c-uri-extension + c-uri-query: unmapped.c-uri-query + c-useragent: unmapped.c-useragent + CallerIpAddress: unmapped.CallerIpAddress + CallerProcessName: unmapped.CallerProcessName + CallingProcessName: unmapped.CallingProcessName + CallTrace: unmapped.CallTrace + Caption: unmapped.Caption + category: unmapped.category + CategoryValue: unmapped.CategoryValue + certificate.serial: unmapped.certificate.serial + CertIssuerName: unmapped.CertIssuerName + CertSerialNumber: unmapped.CertSerialNumber + CertThumbprint: unmapped.CertThumbprint + Channel: unmapped.Channel + cipher: unmapped.cipher + CipherAlgorithm: unmapped.CipherAlgorithm + class_type: unmapped.class_type + ClassName: unmapped.ClassName + client: unmapped.client + client_header_names: unmapped.client_header_names + client.as.number: unmapped.client.as.number + client.as.organization.name: unmapped.client.as.organization.name + client.device: unmapped.client.device + client.domain: unmapped.client.domain + client.id: unmapped.client.id + client.ip: unmapped.client.ip + client.user.full_name: unmapped.client.user.full_name + client.user.id: unmapped.client.user.id + client.zone: unmapped.client.zone + ClientInfoString: unmapped.ClientInfoString + ClientProcessId: unmapped.ClientProcessId + CommandLine: unmapped.CommandLine + ComplianceState: unmapped.ComplianceState + ComputerName: unmapped.ComputerName + ConditionalAccessStatus: unmapped.ConditionalAccessStatus + conn_state: unmapped.conn_state + ConnectionId: unmapped.ConnectionId + ConnectionMode: unmapped.ConnectionMode + CONSUMER: unmapped.CONSUMER + cookie: unmapped.cookie + cs-bytes: unmapped.cs-bytes + cs-cookie: unmapped.cs-cookie + cs-cookie-vars: unmapped.cs-cookie-vars + cs-host: unmapped.cs-host + cs-method: unmapped.cs-method + cs-referrer: unmapped.cs-referrer + cs-version: unmapped.cs-version + CurrentDirectory: unmapped.CurrentDirectory + Data: unmapped.Data + data.protoPayload.methodName: unmapped.data.protoPayload.methodName + data.protoPayload.request.function.timeout: unmapped.data.protoPayload.request.function.timeout + data.protoPayload.serviceData.policyDelta.bindingDeltas{}.action: unmapped.data.protoPayload.serviceData.policyDelta.bindingDeltas{}.action + data.protoPayload.serviceData.policyDelta.bindingDeltas{}.member: unmapped.data.protoPayload.serviceData.policyDelta.bindingDeltas{}.member + data.resource.type: unmapped.data.resource.type + Description: unmapped.Description + desktop_height: unmapped.desktop_height + desktop_width: unmapped.desktop_width + DestAddress: unmapped.DestAddress + Destination: unmapped.Destination + DestinationHostname: unmapped.DestinationHostname + DestinationIp: unmapped.DestinationIp + DestinationPort: unmapped.DestinationPort + DestPort: unmapped.DestPort + Details: unmapped.Details + Device: unmapped.Device + DeviceDescription: unmapped.DeviceDescription + DeviceDetail: unmapped.DeviceDetail + DeviceName: unmapped.DeviceName + DevicesInsights: unmapped.DevicesInsights + DisableIntegrityChecks: unmapped.DisableIntegrityChecks + DisplayName: unmapped.DisplayName + dns-answer: unmapped.dns-answer + dns-query: unmapped.dns-query + dns-record: unmapped.dns-record + DnsHostName: unmapped.DnsHostName + Domain: unmapped.Domain + dst_ip: unmapped.dst_ip + dst_port: unmapped.dst_port + dst-ip: unmapped.dst-ip + dst-port: unmapped.dst-port + EnabledPrivilegeList: unmapped.EnabledPrivilegeList + endpoint: unmapped.endpoint + errorCode: unmapped.errorCode + errorMessage: unmapped.errorMessage + event.action: unmapped.event.action + event.outcome: unmapped.event.outcome + EventData: unmapped.EventData + EventID: unmapped.EventID + eventName: unmapped.eventName + eventSource: unmapped.eventSource + EventType: unmapped.EventType + exe: unmapped.exe + FailureReason: unmapped.FailureReason + Faultingapplicationpath: unmapped.Faultingapplicationpath + fc_request: unmapped.fc_request + FileName: unmapped.FileName + filetype: unmapped.filetype + function: unmapped.function + gcp.audit.method_name: unmapped.gcp.audit.method_name + GrantedAccess: unmapped.GrantedAccess + has_cert_table: unmapped.has_cert_table + Hashes: unmapped.Hashes + history: unmapped.history + HiveName: unmapped.HiveName + host: unmapped.host + Hostname: unmapped.Hostname + http.bodyMagic: unmapped.http.bodyMagic + http.method: unmapped.http.method + id_resp_p: unmapped.id_resp_p + id.orig_h: unmapped.id.orig_h + id.orig_p: unmapped.id.orig_p + id.resp_h: unmapped.id.resp_h + id.resp_p: unmapped.id.resp_p + Image: unmapped.Image + ImageLoaded: unmapped.ImageLoaded + ImageName: unmapped.ImageName + ImagePath: unmapped.ImagePath + inference: unmapped.inference + Initiated: unmapped.Initiated + InitiatedBy: unmapped.InitiatedBy + InterfaceDescription: unmapped.InterfaceDescription + InterfaceGuid: unmapped.InterfaceGuid + IpAddress: unmapped.IpAddress + IpPort: unmapped.IpPort + IsInteractive: unmapped.IsInteractive + Issuer: unmapped.Issuer + jsonPayload.enforcedSecurityPolicy.configuredAction: unmapped.jsonPayload.enforcedSecurityPolicy.configuredAction + jsonPayload.enforcedSecurityPolicy.matchedFieldValue: unmapped.jsonPayload.enforcedSecurityPolicy.matchedFieldValue + KernelDebug: unmapped.KernelDebug + keyboard_layout: unmapped.keyboard_layout + KeyLength: unmapped.KeyLength + LayerRTID: unmapped.LayerRTID + Level: unmapped.Level + LinkName: unmapped.LinkName + local_orig: unmapped.local_orig + local_resp: unmapped.local_resp + LocalName: unmapped.LocalName + LogEntry: unmapped.LogEntry + logged_ssh_inference: unmapped.logged_ssh_inference + LoggedByService: unmapped.LoggedByService + LogonError: unmapped.LogonError + LogonProcessName: unmapped.LogonProcessName + LogonType: unmapped.LogonType + MemberName: unmapped.MemberName + MemberSid: unmapped.MemberSid + message: unmapped.message + method: unmapped.method + methodName: unmapped.methodName + mime_type: unmapped.mime_type + mime_types: unmapped.mime_types + ModifiedProperties: unmapped.ModifiedProperties + ModifyingApplication: unmapped.ModifyingApplication + name: unmapped.name + Namespace: unmapped.Namespace + NetworkLocationDetails: unmapped.NetworkLocationDetails + NewProcessName: unmapped.NewProcessName + NewSd: unmapped.NewSd + NewTargetUserName: unmapped.NewTargetUserName + NewTemplateContent: unmapped.NewTemplateContent + NewUacValue: unmapped.NewUacValue + NewValue: unmapped.NewValue + note: unmapped.note + notice.cipher: unmapped.notice.cipher + notice.password: unmapped.notice.password + notice.request_type: unmapped.notice.request_type + notice.user_agent: unmapped.notice.user_agent + NotificationPackageName: unmapped.NotificationPackageName + object_name: unmapped.object_name + ObjectClass: unmapped.ObjectClass + ObjectDN: unmapped.ObjectDN + ObjectName: unmapped.ObjectName + objectRef.namespace: unmapped.objectRef.namespace + objectRef.resource: unmapped.objectRef.resource + objectRef.subresource: unmapped.objectRef.subresource + ObjectServer: unmapped.ObjectServer + ObjectType: unmapped.ObjectType + ObjectValueName: unmapped.ObjectValueName + OfficeObjectId: unmapped.OfficeObjectId + OfficeWorkload: unmapped.OfficeWorkload + OldTargetUserName: unmapped.OldTargetUserName + OldUacValue: unmapped.OldUacValue + OldValue: unmapped.OldValue + OnexEnabled: unmapped.OnexEnabled + operation: unmapped.operation + OperationName: unmapped.OperationName + OperationNameValue: unmapped.OperationNameValue + OperationType: unmapped.OperationType + orig_bytes: unmapped.orig_bytes + Origin: unmapped.Origin + outcome.reason: unmapped.outcome.reason + param1: unmapped.param1 + param2: unmapped.param2 + Parameters: unmapped.Parameters + parent-domain: unmapped.parent-domain + ParentImage: unmapped.ParentImage + ParentProcessName: unmapped.ParentProcessName + PasswordLastSet: unmapped.PasswordLastSet + path: unmapped.path + PHYType: unmapped.PHYType + PipeName: unmapped.PipeName + PossibleCause: unmapped.PossibleCause + post-body: unmapped.post-body + PreAuthType: unmapped.PreAuthType + PreviousCreationUtcTime: unmapped.PreviousCreationUtcTime + PrivilegeList: unmapped.PrivilegeList + Process_Name: unmapped.Process_Name + ProcessName: unmapped.ProcessName + processPath: unmapped.processPath + ProfileName: unmapped.ProfileName + Properties: unmapped.Properties + properties.message: unmapped.properties.message + proto: unmapped.proto + protoPayload.methodName: unmapped.protoPayload.methodName + protoPayload.request.canIpForward: unmapped.protoPayload.request.canIpForward + protoPayload.serviceName: unmapped.protoPayload.serviceName + Provider: unmapped.Provider + Provider_Name: unmapped.Provider_Name + proxied: unmapped.proxied + QNAME: unmapped.QNAME + qtype_name: unmapped.qtype_name + query: unmapped.query + QueryName: unmapped.QueryName + QueryResults: unmapped.QueryResults + r-dns: unmapped.r-dns + rcode_name: unmapped.rcode_name + RecordType: unmapped.RecordType + referrer: unmapped.referrer + RegistryValue: unmapped.RegistryValue + rejected: unmapped.rejected + related.user: unmapped.related.user + RelativeTargetName: unmapped.RelativeTargetName + RemoteName: unmapped.RemoteName + request_body_len: unmapped.request_body_len + request_type: unmapped.request_type + requestObject.rules.resources: unmapped.requestObject.rules.resources + requestObject.rules.verbs: unmapped.requestObject.rules.verbs + requestObject.spec.containers.image: unmapped.requestObject.spec.containers.image + requestParameters: unmapped.requestParameters + requestParameters.attribute: unmapped.requestParameters.attribute + requestParameters.ipPermissions.items.ipRanges.items.cidrIP: unmapped.requestParameters.ipPermissions.items.ipRanges.items.cidrIP + requestParameters.ipPermissions.items.ipRanges.items.fromPort: unmapped.requestParameters.ipPermissions.items.ipRanges.items.fromPort + requestParameters.userData: unmapped.requestParameters.userData + requestURI: unmapped.requestURI + ResourceDisplayName: unmapped.ResourceDisplayName + ResourceId: unmapped.ResourceId + ResourceIdentity: unmapped.ResourceIdentity + ResourceProviderValue: unmapped.ResourceProviderValue + resp_bytes: unmapped.resp_bytes + resp_mime_types: unmapped.resp_mime_types + response_body_len: unmapped.response_body_len + responseElements.pendingModifiedValues.masterUserPassword: unmapped.responseElements.pendingModifiedValues.masterUserPassword + responseElements.publiclyAccessible: unmapped.responseElements.publiclyAccessible + Result: unmapped.Result + ResultDescription: unmapped.ResultDescription + ResultReason: unmapped.ResultReason + ResultStatus: unmapped.ResultStatus + ResultType: unmapped.ResultType + RiskDetail: unmapped.RiskDetail + sc-status: unmapped.sc-status + SearchFilter: unmapped.SearchFilter + SecurityDescriptor: unmapped.SecurityDescriptor + SecurityID: unmapped.SecurityID + SecurityPackageName: unmapped.SecurityPackageName + server_name: unmapped.server_name + ServerName: unmapped.ServerName + service: unmapped.service + ServiceAccount: unmapped.ServiceAccount + ServiceFileName: unmapped.ServiceFileName + serviceName: unmapped.serviceName + ServicePrincipalNames: unmapped.ServicePrincipalNames + ServiceSid: unmapped.ServiceSid + ServiceStartType: unmapped.ServiceStartType + ServiceType: unmapped.ServiceType + SessionName: unmapped.SessionName + share_name: unmapped.share_name + ShareName: unmapped.ShareName + SidHistory: unmapped.SidHistory + SignatureStatus: unmapped.SignatureStatus + Signed: unmapped.Signed + source: unmapped.source + Source_Name: unmapped.Source_Name + source.domain: unmapped.source.domain + source.ip: unmapped.source.ip + source.user.full_name: unmapped.source.user.full_name + source.user.id: unmapped.source.user.id + SourceAddr: unmapped.SourceAddr + SourceAddress: unmapped.SourceAddress + SourceFileExtension: unmapped.SourceFileExtension + SourceFileName: unmapped.SourceFileName + SourceImage: unmapped.SourceImage + SourceIp: unmapped.SourceIp + SourceItemName: unmapped.SourceItemName + SourceName: unmapped.SourceName + SourcePort: unmapped.SourcePort + SourceSystem: unmapped.SourceSystem + SourceUser: unmapped.SourceUser + src_ip: unmapped.src_ip + src_port: unmapped.src_port + src-ip: unmapped.src-ip + src-port: unmapped.src-port + SSID: unmapped.SSID + stage: unmapped.stage + StartType: unmapped.StartType + statement: unmapped.statement + status: unmapped.status + status_code: unmapped.status_code + Status.errorCode: unmapped.Status.errorCode + Status.failureReason: unmapped.Status.failureReason + StatusText: unmapped.StatusText + subject: unmapped.subject + SubjectDomainName: unmapped.SubjectDomainName + SubjectLogonId: unmapped.SubjectLogonId + SubjectUserName: unmapped.SubjectUserName + SubjectUserSid: unmapped.SubjectUserSid + SubStatus: unmapped.SubStatus + subsystem: unmapped.subsystem + successful: unmapped.successful + TargetDomainName: unmapped.TargetDomainName + TargetImage: unmapped.TargetImage + TargetInfo: unmapped.TargetInfo + TargetLogonId: unmapped.TargetLogonId + TargetName: unmapped.TargetName + TargetResources: unmapped.TargetResources + TargetServerName: unmapped.TargetServerName + TargetSid: unmapped.TargetSid + TargetUser: unmapped.TargetUser + TargetUserName: unmapped.TargetUserName + TargetUserSid: unmapped.TargetUserSid + TaskContent: unmapped.TaskContent + TaskName: unmapped.TaskName + TemplateContent: unmapped.TemplateContent + terminatingRuleId: unmapped.terminatingRuleId + TestSigning: unmapped.TestSigning + TicketEncryptionType: unmapped.TicketEncryptionType + TicketOptions: unmapped.TicketOptions + till: unmapped.till + TokenIssuerType: unmapped.TokenIssuerType + total_bytes: unmapped.total_bytes + TransmittedServices: unmapped.TransmittedServices + Type: unmapped.Type + uri: unmapped.uri + User: unmapped.User + user_agent: unmapped.user_agent + user_agent.original: unmapped.user_agent.original + user.groups: unmapped.user.groups + user.username: unmapped.user.username + UserAccountControl: unmapped.UserAccountControl + UserAgent: unmapped.UserAgent + userAgent.browser: unmapped.userAgent.browser + userAgent.os: unmapped.userAgent.os + UserContext: unmapped.UserContext + UserID: unmapped.UserID + UserName: unmapped.UserName + UserPrincipalName: unmapped.UserPrincipalName + UsersInsights: unmapped.UsersInsights + UsersInsights.IsDormantAccount: unmapped.UsersInsights.IsDormantAccount + UsersInsights.IsNewAccount: unmapped.UsersInsights.IsNewAccount + validation_status: unmapped.validation_status + Value: unmapped.Value + verb: unmapped.verb + Workstation: unmapped.Workstation + WorkstationName: unmapped.WorkstationName From 0c64c329f0f1d36c6eda3123c32073995b17dae8 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:33:10 +0200 Subject: [PATCH 11/15] gis-9415 add global alternative mapping --- uncoder-core/app/translator/core/mapping.py | 15 ++++++++++----- .../app/translator/core/models/query_container.py | 2 +- .../global_alternative/ocsf/default.yml | 1 - .../app/translator/platforms/splunk/mapping.py | 8 ++++++-- 4 files changed, 17 insertions(+), 9 deletions(-) rename uncoder-core/app/translator/mappings/{ => platforms}/global_alternative/ocsf/default.yml (99%) diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index b99bee54..9348f1ad 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -1,7 +1,8 @@ from __future__ import annotations +import os from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Optional, TypeVar, Union +from typing import TYPE_CHECKING, ClassVar, Optional, TypeVar, Union from app.translator.core.exceptions.core import ( StrictPlatformException, @@ -16,6 +17,7 @@ DEFAULT_MAPPING_NAME = "default" +GLOBAL_ALTERNATIVE_MAPPING_DIR = "global_alternative" class LogSourceSignature(ABC): @@ -113,14 +115,15 @@ class BasePlatformMappings: is_strict_mapping: bool = False skip_load_default_mappings: bool = True extend_default_mapping_with_all_fields: bool = False - global_mappings: list[str] = [] + global_alternative_mappings: ClassVar[list[str]] = [] def __init__(self, platform_dir: str, platform_details: PlatformDetails): self._loader = LoaderFileMappings() self.details = platform_details self._source_mappings = self.prepare_mapping(platform_dir) self._alternative_mappings = self.prepare_alternative_mapping(platform_dir) - global_alternative_mappings = self.prepare_global_alternative_mapping() + if self.global_alternative_mappings: + self._alternative_mappings.update(self.prepare_global_alternative_mapping()) def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None: default_mapping.fields_mapping.update(fields_mapping) @@ -132,8 +135,10 @@ def prepare_alternative_mapping(self, platform_dir: str) -> dict[str, dict[str, return alternative_mappings def prepare_global_alternative_mapping(self) -> dict[str, dict[str, SourceMapping]]: - globa_alternative_mappings = {} - return globa_alternative_mappings + global_alternative_mappings = {} + for name in self.global_alternative_mappings: + global_alternative_mappings[name] = self.prepare_mapping(os.path.join(GLOBAL_ALTERNATIVE_MAPPING_DIR, name)) + return global_alternative_mappings def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: source_mappings = {} diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index c35804b2..854c2972 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -136,7 +136,7 @@ class RawQueryContainer: class RawQueryDictContainer: query: dict language: str - meta_info: dict + meta_info: MetaInfoContainer = field(default_factory=MetaInfoContainer) @dataclass diff --git a/uncoder-core/app/translator/mappings/global_alternative/ocsf/default.yml b/uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml similarity index 99% rename from uncoder-core/app/translator/mappings/global_alternative/ocsf/default.yml rename to uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml index 361135e3..351c921c 100644 --- a/uncoder-core/app/translator/mappings/global_alternative/ocsf/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml @@ -1,7 +1,6 @@ platform: Global OCSF source: default -default_log_source: {} fieldmappings: IntegrityLevel: process.integrity diff --git a/uncoder-core/app/translator/platforms/splunk/mapping.py b/uncoder-core/app/translator/platforms/splunk/mapping.py index b5750532..cff045ec 100644 --- a/uncoder-core/app/translator/platforms/splunk/mapping.py +++ b/uncoder-core/app/translator/platforms/splunk/mapping.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import ClassVar, Optional from app.translator.core.mapping import BasePlatformMappings, LogSourceSignature from app.translator.platforms.splunk.const import splunk_alert_details, splunk_query_details @@ -39,9 +39,13 @@ def __str__(self) -> str: class SplunkMappings(BasePlatformMappings): + global_alternative_mappings: ClassVar[list[str]] = ["ocsf"] + def prepare_log_source_signature(self, mapping: dict) -> SplunkLogSourceSignature: log_source = mapping.get("log_source", {}) - default_log_source = mapping["default_log_source"] + default_log_source = ( + mapping.get("default_log_source") if mapping.get("default_log_source") else {"source": "WinEventLog: *"} + ) return SplunkLogSourceSignature( sources=log_source.get("source"), source_types=log_source.get("sourcetype"), From b7717c8e82540de4d6d3857f67104088204991e5 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Fri, 24 Jan 2025 12:15:15 +0200 Subject: [PATCH 12/15] gis-9415 fix --- .../mappings/platforms/global_alternative/ocsf/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml b/uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml index 351c921c..fad3735c 100644 --- a/uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/global_alternative/ocsf/default.yml @@ -2,7 +2,7 @@ platform: Global OCSF source: default -fieldmappings: +field_mapping: IntegrityLevel: process.integrity ParentCommandLine: - process.parent_process.cmd_line From 9e5e17410726a7db83562caca16bc96b5eda6e97 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:35:54 +0200 Subject: [PATCH 13/15] gis-9415 fix --- uncoder-core/app/translator/core/mapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 9348f1ad..3bacd6af 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -147,7 +147,7 @@ def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) if (source_id := mapping_dict["source"]) == DEFAULT_MAPPING_NAME: default_mapping.log_source_signature = log_source_signature - if self.skip_load_default_mappings: + if self.skip_load_default_mappings and not platform_dir.startswith(GLOBAL_ALTERNATIVE_MAPPING_DIR): continue field_mappings_dict = mapping_dict.get("field_mapping", {}) @@ -164,7 +164,7 @@ def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]: conditions=conditions, ) - if self.skip_load_default_mappings: + if self.skip_load_default_mappings and not platform_dir.startswith(GLOBAL_ALTERNATIVE_MAPPING_DIR): source_mappings[DEFAULT_MAPPING_NAME] = default_mapping if self.extend_default_mapping_with_all_fields: From 3cc30bbb08020df5db0042538b2d0a644b06805a Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Thu, 30 Jan 2025 13:00:32 +0200 Subject: [PATCH 14/15] TDM-9415 enable one vendor translation with alt_mapping --- uncoder-core/app/translator/translator.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/uncoder-core/app/translator/translator.py b/uncoder-core/app/translator/translator.py index 13231bd5..80a35b58 100644 --- a/uncoder-core/app/translator/translator.py +++ b/uncoder-core/app/translator/translator.py @@ -152,13 +152,7 @@ def translate_one( source_alt_mapping: Optional[str] = None, target_alt_mapping: Optional[str] = None, ) -> (bool, str): - if source == target: - if target_alt_mapping or source_alt_mapping: - message = ( - "Currently, Uncoder doesn't support translation between " - "non-default data schemas of the same platform." - ) - return False, message + if source == target and not (target_alt_mapping or source_alt_mapping): return True, text return self.__translate_one( text=text, From 0766f1c17cac6a2ee50ba87c9596c128b63b3e08 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Fri, 31 Jan 2025 10:46:54 +0200 Subject: [PATCH 15/15] TDM-9415 fixes --- .../translator/platforms/base/aql/parsers/aql.py | 12 ++++++++---- .../platforms/base/lucene/parsers/lucene.py | 14 ++++++++++---- .../translator/platforms/base/spl/parsers/spl.py | 6 +++++- .../translator/platforms/base/sql/parsers/sql.py | 16 +++++++++++----- .../platforms/chronicle/parsers/chronicle.py | 6 +++++- .../elasticsearch/parsers/elasticsearch_eql.py | 6 +++++- .../platforms/logscale/parsers/logscale.py | 6 +++++- 7 files changed, 49 insertions(+), 17 deletions(-) diff --git a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py index 0dad8283..0881459d 100644 --- a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py @@ -17,7 +17,7 @@ """ import re -from typing import Union +from typing import Optional, Union from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.models.functions.base import ParsedFunctions @@ -105,8 +105,8 @@ def __parse_log_sources(self, query: str) -> tuple[dict[str, Union[list[str], li return log_sources, query - def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[int]]], ParsedFunctions]: - query = self.__clean_query(text) + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + query = self.__clean_query(query) self.__check_table(query) query, functions = self.platform_functions.parse(query) log_sources, query = self.__parse_log_sources(query) @@ -118,7 +118,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( query_tokens, functions.functions ) - source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens + function_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.function_fields = function_field_tokens diff --git a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py index 49f05c98..3a235ff9 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py @@ -17,7 +17,9 @@ """ import re +from typing import Optional, Union +from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.lucene.tokenizer import LuceneTokenizer @@ -31,7 +33,7 @@ class LuceneQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: log_sources = {} for source_type in self.log_source_key_types: pattern = self.log_source_pattern.replace("___source_type___", source_type) @@ -43,13 +45,17 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: pos_end = search.end() query = query[:pos_start] + query[pos_end:] - return query, log_sources + return query, log_sources, None def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - query, log_sources = self._parse_query(raw_query_container.query) + query, log_sources, _ = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(query_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] diff --git a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py index f56af913..fe62c227 100644 --- a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py @@ -75,7 +75,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( query_tokens, functions.functions ) - source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens + function_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.function_fields = function_field_tokens diff --git a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py index 01be3500..1fd62058 100644 --- a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py @@ -17,7 +17,9 @@ """ import re +from typing import Optional, Union +from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.sql.tokenizer import SqlTokenizer @@ -30,21 +32,25 @@ class SqlQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*--.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: log_source = {"table": []} if re.search(self.query_delimiter_pattern, query, flags=re.IGNORECASE): table_search = re.search(self.table_pattern, query) table = table_search.group("table") log_source["table"] = [table] - return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source + return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source, None - return query, log_source + return query, log_source, None def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - query, log_sources = self._parse_query(raw_query_container.query) + query, log_sources, _ = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(query_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py index 0cc1af82..e0af8a22 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py @@ -36,7 +36,11 @@ class ChronicleQueryParser(PlatformQueryParser): def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query_tokens = self.get_query_tokens(raw_query_container.query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(query_field_tokens, {}) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens, + log_sources={}, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py index 377b1e08..95c4e0f4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py @@ -30,7 +30,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query, log_sources = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(query_field_tokens, log_sources) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens, + log_sources=log_sources, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py index ddf2fcd1..fbb949ce 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py @@ -46,7 +46,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( query_tokens, functions.functions ) - source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, {}) + source_mappings = self.get_source_mappings( + field_tokens=query_field_tokens + function_field_tokens, + log_sources={}, + alt_mapping=raw_query_container.meta_info.source_alt_mapping, + ) meta_info = raw_query_container.meta_info meta_info.query_fields = query_field_tokens meta_info.function_fields = function_field_tokens pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy