diff --git a/uncoder-core/app/translator/core/exceptions/core.py b/uncoder-core/app/translator/core/exceptions/core.py index 47810576..8a5256e6 100644 --- a/uncoder-core/app/translator/core/exceptions/core.py +++ b/uncoder-core/app/translator/core/exceptions/core.py @@ -10,19 +10,14 @@ class BasePlatformException(BaseException): class StrictPlatformException(BasePlatformException): - field_name: str = None - - def __init__( - self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None - ): + def __init__(self, platform_name: str, fields: list[str], mapping: Optional[str] = None): message = ( f"Platform {platform_name} has strict mapping. " - f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping." + f"Source fields: {', '.join(fields)} have no mapping." f" Mapping file: {mapping}." if mapping else "" ) - self.field_name = field_name super().__init__(message) diff --git a/uncoder-core/app/translator/core/functions.py b/uncoder-core/app/translator/core/functions.py index 2517129b..d154ab1b 100644 --- a/uncoder-core/app/translator/core/functions.py +++ b/uncoder-core/app/translator/core/functions.py @@ -94,17 +94,16 @@ def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRe def render(self, function: Function, source_mapping: SourceMapping) -> str: raise NotImplementedError - @staticmethod - def map_field(field: Union[Alias, Field], source_mapping: SourceMapping) -> str: + def map_field(self, field: Union[Alias, Field], source_mapping: SourceMapping) -> str: if isinstance(field, Alias): return field.name - generic_field_name = field.get_generic_field_name(source_mapping.source_id) - mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name) - if isinstance(mapped_field, list): - mapped_field = mapped_field[0] + if isinstance(field, Field): + mappings = self.manager.platform_functions.platform_query_render.mappings + mapped_fields = mappings.map_field(field, source_mapping) + return mapped_fields[0] - return mapped_field if mapped_field else field.source_name + raise NotSupportedFunctionException class PlatformFunctionsManager: diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index bdab5f6d..78bf8b9f 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -1,10 +1,16 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Optional, TypeVar +from typing import TYPE_CHECKING, Optional, TypeVar +from app.translator.core.exceptions.core import StrictPlatformException +from app.translator.core.models.platform_details import PlatformDetails from app.translator.mappings.utils.load_from_files import LoaderFileMappings +if TYPE_CHECKING: + from app.translator.core.models.query_tokens.field import Field + + DEFAULT_MAPPING_NAME = "default" @@ -85,12 +91,16 @@ def __init__( class BasePlatformMappings: + details: PlatformDetails = None + + is_strict_mapping: bool = False skip_load_default_mappings: bool = True extend_default_mapping_with_all_fields: bool = False - def __init__(self, platform_dir: str): + def __init__(self, platform_dir: str, platform_details: PlatformDetails): self._loader = LoaderFileMappings() self._platform_dir = platform_dir + self.details = platform_details self._source_mappings = self.prepare_mapping() def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None: @@ -148,6 +158,32 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]: def default_mapping(self) -> SourceMapping: return self._source_mappings[DEFAULT_MAPPING_NAME] + def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]: + unmapped = [] + for field in field_tokens: + generic_field_name = field.get_generic_field_name(source_mapping.source_id) + mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name) + if not mapped_field and field.source_name not in unmapped: + unmapped.append(field.source_name) + + if self.is_strict_mapping and unmapped: + raise StrictPlatformException( + platform_name=self.details.name, fields=unmapped, mapping=source_mapping.source_id + ) + + return unmapped + + @staticmethod + def map_field(field: Field, source_mapping: SourceMapping) -> list[str]: + generic_field_name = field.get_generic_field_name(source_mapping.source_id) + # field can be mapped to corresponding platform field name or list of platform field names + mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name) + + if isinstance(mapped_field, str): + mapped_field = [mapped_field] + + return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name] + class BaseCommonPlatformMappings(ABC, BasePlatformMappings): def prepare_mapping(self) -> dict[str, SourceMapping]: diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 618f2d37..6158b679 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -184,6 +184,7 @@ class QueryRender(ABC): details: PlatformDetails = None is_single_line_comment: bool = False unsupported_functions_text = "Unsupported functions were excluded from the result query:" + unmapped_fields_text = "Unmapped fields: " platform_functions: PlatformFunctions = None @@ -206,6 +207,11 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions: return query + def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[str]]) -> str: + if fields: + return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{', '.join(fields)}") + return query + def wrap_with_comment(self, value: str) -> str: return f"{self.comment_symbol} {value}" @@ -216,7 +222,6 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta class PlatformQueryRender(QueryRender): mappings: BasePlatformMappings = None - is_strict_mapping: bool = False or_token = "or" and_token = "and" @@ -247,22 +252,10 @@ def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], fu def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions: return self.platform_functions.render(functions, source_mapping) - def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]: - generic_field_name = field.get_generic_field_name(source_mapping.source_id) - # field can be mapped to corresponding platform field name or list of platform field names - mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name) - if not mapped_field and self.is_strict_mapping: - raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name) - - if isinstance(mapped_field, str): - mapped_field = [mapped_field] - - return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name] - def map_predefined_field(self, predefined_field: PredefinedField) -> str: if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)): - if self.is_strict_mapping: - raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name) + if self.mappings.is_strict_mapping: + raise StrictPlatformException(platform_name=self.details.name, fields=[predefined_field.name]) return predefined_field.name @@ -275,7 +268,7 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> elif token.predefined_field: mapped_fields = [self.map_predefined_field(token.predefined_field)] else: - mapped_fields = self.map_field(token.field, source_mapping) + mapped_fields = self.mappings.map_field(token.field, source_mapping) joined = self.logical_operators_map[LogicalOperatorType.OR].join( [ self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) @@ -285,9 +278,13 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> return self.group_token % joined if len(mapped_fields) > 1 else joined if isinstance(token, FieldField): alias_left, field_left = token.alias_left, token.field_left - mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping) + mapped_fields_left = ( + [alias_left.name] if alias_left else self.mappings.map_field(field_left, source_mapping) + ) alias_right, field_right = token.alias_right, token.field_right - mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping) + mapped_fields_right = ( + [alias_right.name] if alias_right else self.mappings.map_field(field_right, source_mapping) + ) cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right)) joined = self.logical_operators_map[LogicalOperatorType.OR].join( [ @@ -311,14 +308,9 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str: result_values = [] - unmapped_fields = set() for token in tokens: - try: - result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) - except StrictPlatformException as err: - unmapped_fields.add(err.field_name) - if unmapped_fields: - raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields)) + result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) + return "".join(result_values) def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str: @@ -351,11 +343,13 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: query = self._join_query_parts(prefix, query, functions) query = self.wrap_with_meta_info(query, meta_info) + query = self.wrap_with_unmapped_fields(query, unmapped_fields) return self.wrap_with_not_supported_functions(query, not_supported_functions) @staticmethod @@ -417,8 +411,10 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap mapped_field = source_mapping.fields_mapping.get_platform_field_name( generic_field_name=generic_field_name ) - if not mapped_field and self.is_strict_mapping: - raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name) + if not mapped_field and self.mappings.is_strict_mapping: + raise StrictPlatformException( + platform_name=self.details.name, fields=[field.source_name], mapping=source_mapping.source_id + ) if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping): for prefix in prefix_list: if prefix not in defined_raw_log_fields: @@ -428,6 +424,9 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap def _generate_from_tokenized_query_container_by_source_mapping( self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping ) -> str: + unmapped_fields = self.mappings.check_fields_mapping_existence( + query_container.meta_info.query_fields, source_mapping + ) rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix) @@ -443,6 +442,7 @@ def _generate_from_tokenized_query_container_by_source_mapping( query=query, functions=rendered_functions.rendered, not_supported_functions=not_supported_functions, + unmapped_fields=unmapped_fields, meta_info=query_container.meta_info, source_mapping=source_mapping, ) diff --git a/uncoder-core/app/translator/platforms/athena/const.py b/uncoder-core/app/translator/platforms/athena/const.py index 1f286117..db261b69 100644 --- a/uncoder-core/app/translator/platforms/athena/const.py +++ b/uncoder-core/app/translator/platforms/athena/const.py @@ -9,4 +9,4 @@ "alt_platform_name": "OCSF", } -athena_details = PlatformDetails(**ATHENA_QUERY_DETAILS) +athena_query_details = PlatformDetails(**ATHENA_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/athena/mapping.py b/uncoder-core/app/translator/platforms/athena/mapping.py index ab33bd7a..d15d5156 100644 --- a/uncoder-core/app/translator/platforms/athena/mapping.py +++ b/uncoder-core/app/translator/platforms/athena/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.athena.const import athena_query_details class AthenaLogSourceSignature(LogSourceSignature): @@ -40,4 +41,4 @@ def get_suitable_source_mappings(self, field_names: list[str], table: Optional[s return suitable_source_mappings -athena_mappings = AthenaMappings(platform_dir="athena") +athena_query_mappings = AthenaMappings(platform_dir="athena", platform_details=athena_query_details) diff --git a/uncoder-core/app/translator/platforms/athena/parsers/athena.py b/uncoder-core/app/translator/platforms/athena/parsers/athena.py index 565f4165..9e2bd555 100644 --- a/uncoder-core/app/translator/platforms/athena/parsers/athena.py +++ b/uncoder-core/app/translator/platforms/athena/parsers/athena.py @@ -18,14 +18,14 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager -from app.translator.platforms.athena.const import athena_details -from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings +from app.translator.platforms.athena.const import athena_query_details +from app.translator.platforms.athena.mapping import AthenaMappings, athena_query_mappings from app.translator.platforms.base.sql.parsers.sql import SqlQueryParser @parser_manager.register_supported_by_roota class AthenaQueryParser(SqlQueryParser): - details: PlatformDetails = athena_details - mappings: AthenaMappings = athena_mappings + details: PlatformDetails = athena_query_details + mappings: AthenaMappings = athena_query_mappings query_delimiter_pattern = r"\sFROM\s\S*\sWHERE\s" table_pattern = r"\sFROM\s(?P[a-zA-Z\.\-\*]+)\sWHERE\s" diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena.py b/uncoder-core/app/translator/platforms/athena/renders/athena.py index 8550c94a..2b431af2 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena.py @@ -19,19 +19,19 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.athena.const import athena_details -from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings +from app.translator.platforms.athena.const import athena_query_details +from app.translator.platforms.athena.mapping import AthenaMappings, athena_query_mappings from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender class AthenaFieldValueRender(SqlFieldValueRender): - details: PlatformDetails = athena_details + details: PlatformDetails = athena_query_details @render_manager.register class AthenaQueryRender(SqlQueryRender): - details: PlatformDetails = athena_details - mappings: AthenaMappings = athena_mappings + details: PlatformDetails = athena_query_details + mappings: AthenaMappings = athena_query_mappings or_token = "OR" diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena_cti.py b/uncoder-core/app/translator/platforms/athena/renders/athena_cti.py index aa4f986b..c46290e8 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena_cti.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena_cti.py @@ -20,13 +20,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render_cti import RenderCTI from app.translator.managers import render_cti_manager -from app.translator.platforms.athena.const import athena_details +from app.translator.platforms.athena.const import athena_query_details from app.translator.platforms.athena.mappings.athena_cti import DEFAULT_ATHENA_MAPPING @render_cti_manager.register class AthenaCTI(RenderCTI): - details: PlatformDetails = athena_details + details: PlatformDetails = athena_query_details field_value_template: str = "{key} = '{value}'" or_operator: str = " OR " diff --git a/uncoder-core/app/translator/platforms/base/aql/mapping.py b/uncoder-core/app/translator/platforms/base/aql/mapping.py index c0fb4b2f..a975a1b4 100644 --- a/uncoder-core/app/translator/platforms/base/aql/mapping.py +++ b/uncoder-core/app/translator/platforms/base/aql/mapping.py @@ -90,6 +90,3 @@ def get_suitable_source_mappings( suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]] return suitable_source_mappings - - -aql_mappings = AQLMappings(platform_dir="qradar") diff --git a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py index 4bc3f46a..8d6fc601 100644 --- a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py @@ -26,14 +26,12 @@ from app.translator.platforms.base.aql.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, TABLE_GROUP_PATTERN from app.translator.platforms.base.aql.functions import AQLFunctions, aql_functions from app.translator.platforms.base.aql.log_source_map import LOG_SOURCE_FUNCTIONS_MAP -from app.translator.platforms.base.aql.mapping import AQLMappings, aql_mappings from app.translator.platforms.base.aql.tokenizer import AQLTokenizer from app.translator.tools.utils import get_match_group class AQLQueryParser(PlatformQueryParser): tokenizer: AQLTokenizer = AQLTokenizer(aql_functions) - mappings: AQLMappings = aql_mappings platform_functions: AQLFunctions = aql_functions log_source_functions = ("LOGSOURCENAME", "LOGSOURCEGROUPNAME") diff --git a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py index 6c0c1665..58fbc3ff 100644 --- a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py @@ -23,7 +23,7 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue -from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings +from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager @@ -121,8 +121,6 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: class AQLQueryRender(PlatformQueryRender): - mappings: AQLMappings = aql_mappings - or_token = "OR" and_token = "AND" not_token = "NOT" diff --git a/uncoder-core/app/translator/platforms/chronicle/mapping.py b/uncoder-core/app/translator/platforms/chronicle/mapping.py index bea60c0e..d341eef8 100644 --- a/uncoder-core/app/translator/platforms/chronicle/mapping.py +++ b/uncoder-core/app/translator/platforms/chronicle/mapping.py @@ -1,4 +1,5 @@ from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.chronicle.const import chronicle_query_details, chronicle_rule_details class ChronicleLogSourceSignature(LogSourceSignature): @@ -10,6 +11,8 @@ def __str__(self) -> str: class ChronicleMappings(BasePlatformMappings): + is_strict_mapping = True + def prepare_log_source_signature(self, mapping: dict) -> ChronicleLogSourceSignature: ... @@ -28,4 +31,5 @@ def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMap return suitable_source_mappings -chronicle_mappings = ChronicleMappings(platform_dir="chronicle") +chronicle_query_mappings = ChronicleMappings(platform_dir="chronicle", platform_details=chronicle_query_details) +chronicle_rule_mappings = ChronicleMappings(platform_dir="chronicle", platform_details=chronicle_rule_details) diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py index b36d1197..7c50cb06 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py @@ -21,13 +21,13 @@ from app.translator.core.parser import PlatformQueryParser from app.translator.managers import parser_manager from app.translator.platforms.chronicle.const import chronicle_query_details -from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings +from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_query_mappings from app.translator.platforms.chronicle.tokenizer import ChronicleQueryTokenizer @parser_manager.register_supported_by_roota class ChronicleQueryParser(PlatformQueryParser): - mappings: ChronicleMappings = chronicle_mappings + mappings: ChronicleMappings = chronicle_query_mappings tokenizer: ChronicleQueryTokenizer = ChronicleQueryTokenizer() details: PlatformDetails = chronicle_query_details diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py index c7929714..888b55eb 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py @@ -23,7 +23,7 @@ from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager from app.translator.platforms.chronicle.const import chronicle_rule_details -from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings +from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_rule_mappings from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser from app.translator.platforms.chronicle.tokenizer import ChronicleRuleTokenizer @@ -35,7 +35,7 @@ class ChronicleRuleParser(ChronicleQueryParser): meta_info_pattern = "meta:\n(?P[a-zA-Z0-9_\\\.*,>–<—~#$’`:;%+^\|?!@\s\"/=\-&'\(\)\[\]]+)\n\s+events:" # noqa: RUF001 rule_pattern = "events:\n\s*(?P[a-zA-Z\w0-9_%{}\|\.,!#^><:~\s\"\/=+?\-–&;$()`\*@\[\]'\\\]+)\n\s+condition:" # noqa: RUF001 event_name_pattern = "condition:\n\s*(?P\$[a-zA-Z_0-9]+)\n" - mappings: ChronicleMappings = chronicle_mappings + mappings: ChronicleMappings = chronicle_rule_mappings tokenizer = ChronicleRuleTokenizer() def __parse_rule(self, rule: str) -> tuple[str, str, str]: diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py index 8bcbe56f..7642929f 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py @@ -27,7 +27,7 @@ from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import chronicle_query_details from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager -from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings +from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_query_mappings class ChronicleFieldValueRender(BaseFieldValueRender): @@ -101,9 +101,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 @render_manager.register class ChronicleQueryRender(PlatformQueryRender): details: PlatformDetails = chronicle_query_details - mappings: ChronicleMappings = chronicle_mappings - - is_strict_mapping = True + mappings: ChronicleMappings = chronicle_query_mappings or_token = "or" and_token = "and" diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py index 1961c72b..3f59f42b 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py @@ -26,6 +26,7 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details +from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_rule_mappings from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValueRender, ChronicleQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated Chronicle Security rule." @@ -84,6 +85,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @render_manager.register class ChronicleSecurityRuleRender(ChronicleQueryRender): details: PlatformDetails = chronicle_rule_details + mappings: ChronicleMappings = chronicle_rule_mappings or_token = "or" field_value_render = ChronicleRuleFieldValueRender(or_token=or_token) @@ -108,7 +110,8 @@ def finalize_query( functions: str, meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 - not_supported_functions: Optional[list] = None, # noqa: ARG002, + not_supported_functions: Optional[list] = None, # , + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -124,4 +127,6 @@ def finalize_query( rule = rule.replace("", meta_info.status) rule = rule.replace("", ", ".join(meta_info.false_positives)) rule = rule.replace("", ", ".join(meta_info.tags)) - return rule.replace("", str(meta_info.date)) + rule = rule.replace("", str(meta_info.date)) + rule = self.wrap_with_unmapped_fields(rule, unmapped_fields) + return self.wrap_with_not_supported_functions(rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/crowdstrike/mapping.py b/uncoder-core/app/translator/platforms/crowdstrike/mapping.py index 80de46a3..5c41399b 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/mapping.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.crowdstrike.const import crowdstrike_query_details class CrowdStrikeLogSourceSignature(LogSourceSignature): @@ -38,4 +39,4 @@ def get_suitable_source_mappings(self, field_names: list[str], event_simpleName: return suitable_source_mappings or [self._source_mappings[DEFAULT_MAPPING_NAME]] -crowdstrike_mappings = CrowdstrikeMappings(platform_dir="crowdstrike") +crowdstrike_query_mappings = CrowdstrikeMappings(platform_dir="crowdstrike", platform_details=crowdstrike_query_details) diff --git a/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py index 80130636..08ec0b7f 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py @@ -21,7 +21,7 @@ from app.translator.platforms.base.spl.parsers.spl import SplQueryParser from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions -from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings +from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_query_mappings @parser_manager.register_supported_by_roota @@ -31,7 +31,7 @@ class CrowdStrikeQueryParser(SplQueryParser): log_source_pattern = r"___source_type___\s*=\s*(?:\"(?P[%a-zA-Z_*:0-9\-/]+)\"|(?P[%a-zA-Z_*:0-9\-/]+))(?:\s+(?:and|or)\s+|\s+)?" # noqa: E501 log_source_key_types = ("event_simpleName",) - mappings: CrowdstrikeMappings = crowdstrike_mappings + mappings: CrowdstrikeMappings = crowdstrike_query_mappings platform_functions: CrowdStrikeFunctions = crowd_strike_functions wrapped_with_comment_pattern = r"^\s*`(?:|\n|.)*`" diff --git a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py index 3e5900cc..40911708 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py @@ -22,7 +22,7 @@ from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions -from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings +from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_query_mappings class CrowdStrikeFieldValueRender(SplFieldValueRender): @@ -32,7 +32,7 @@ class CrowdStrikeFieldValueRender(SplFieldValueRender): @render_manager.register class CrowdStrikeQueryRender(SplQueryRender): details: PlatformDetails = crowdstrike_query_details - mappings: CrowdstrikeMappings = crowdstrike_mappings + mappings: CrowdstrikeMappings = crowdstrike_query_mappings platform_functions: CrowdStrikeFunctions = None or_token = "OR" diff --git a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py index 6c71ab29..b0489fbf 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py @@ -1,8 +1,16 @@ from app.translator.platforms.base.lucene.mapping import LuceneMappings +from app.translator.platforms.elasticsearch.const import ( + elastalert_details, + elasticsearch_lucene_query_details, + elasticsearch_rule_details, + kibana_rule_details, + xpack_watcher_details, +) - -class ElasticSearchMappings(LuceneMappings): - pass - - -elasticsearch_mappings = ElasticSearchMappings(platform_dir="elasticsearch") +elasticsearch_lucene_query_mappings = LuceneMappings( + platform_dir="elasticsearch", platform_details=elasticsearch_lucene_query_details +) +elasticsearch_rule_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=elasticsearch_rule_details) +elastalert_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=elastalert_details) +kibana_rule_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=kibana_rule_details) +xpack_watcher_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=xpack_watcher_details) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch.py index a3bad851..2f287a93 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch.py @@ -18,12 +18,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.parsers.lucene import LuceneQueryParser from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import elasticsearch_lucene_query_mappings @parser_manager.register_supported_by_roota class ElasticSearchQueryParser(LuceneQueryParser): details: PlatformDetails = elasticsearch_lucene_query_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = elasticsearch_lucene_query_mappings diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py index 0b7b20c4..6904e47b 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -26,8 +26,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_DETECTION_RULE, elasticsearch_rule_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import elasticsearch_rule_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( ElasticSearchFieldValue, ElasticSearchQueryRender, @@ -43,7 +44,7 @@ class ElasticSearchRuleFieldValue(ElasticSearchFieldValue): @render_manager.register class ElasticSearchRuleRender(ElasticSearchQueryRender): details: PlatformDetails = elasticsearch_rule_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = elasticsearch_rule_mappings mitre: MitreConfig = MitreConfig() or_token = "OR" @@ -86,6 +87,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -109,4 +111,5 @@ def finalize_query( } ) rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py index 9d7914ab..6b28a9e3 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py @@ -24,8 +24,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_ALERT, elastalert_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import elastalert_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( ElasticSearchFieldValue, ElasticSearchQueryRender, @@ -43,7 +44,7 @@ class ElasticAlertRuleFieldValue(ElasticSearchFieldValue): @render_manager.register class ElastAlertRuleRender(ElasticSearchQueryRender): details: PlatformDetails = elastalert_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = elastalert_mappings or_token = "OR" and_token = "AND" @@ -59,6 +60,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -75,4 +77,5 @@ def finalize_query( ) rule = rule.replace("", meta_info.title or _AUTOGENERATED_TEMPLATE) rule = rule.replace("", _SEVERITIES_MAP[meta_info.severity]) + rule = self.wrap_with_unmapped_fields(rule, unmapped_fields) return self.wrap_with_not_supported_functions(rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py index 2e6a12f0..817707ae 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py @@ -19,9 +19,10 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import elasticsearch_lucene_query_mappings class ElasticSearchFieldValue(LuceneFieldValueRender): @@ -31,7 +32,7 @@ class ElasticSearchFieldValue(LuceneFieldValueRender): @render_manager.register class ElasticSearchQueryRender(LuceneQueryRender): details: PlatformDetails = elasticsearch_lucene_query_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = elasticsearch_lucene_query_mappings or_token = "OR" field_value_render = ElasticSearchFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py index 53a4acf5..e799bdfe 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py @@ -25,8 +25,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import KIBANA_RULE, KIBANA_SEARCH_SOURCE_JSON, kibana_rule_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import kibana_rule_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( ElasticSearchFieldValue, ElasticSearchQueryRender, @@ -43,7 +44,7 @@ class KibanaFieldValue(ElasticSearchFieldValue): @render_manager.register class KibanaRuleRender(ElasticSearchQueryRender): details: PlatformDetails = kibana_rule_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = kibana_rule_mappings or_token = "OR" field_value_render = KibanaFieldValue(or_token=or_token) @@ -55,6 +56,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -74,4 +76,5 @@ def finalize_query( references=meta_info.references, ) rule_str = json.dumps(rule, indent=4, sort_keys=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py index d8421977..eab58aa4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py @@ -25,8 +25,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import XPACK_WATCHER_RULE, xpack_watcher_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import xpack_watcher_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( ElasticSearchFieldValue, ElasticSearchQueryRender, @@ -43,7 +44,7 @@ class XpackWatcherRuleFieldValue(ElasticSearchFieldValue): @render_manager.register class XPackWatcherRuleRender(ElasticSearchQueryRender): details: PlatformDetails = xpack_watcher_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = xpack_watcher_mappings or_token = "OR" field_value_render = XpackWatcherRuleFieldValue(or_token=or_token) @@ -55,6 +56,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -78,4 +80,5 @@ def finalize_query( rule["input"]["search"]["request"]["indices"] = indices rule["actions"]["send_email"]["email"]["subject"] = meta_info.title or _AUTOGENERATED_TEMPLATE rule_str = json.dumps(rule, indent=4, sort_keys=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/forti_siem/mapping.py b/uncoder-core/app/translator/platforms/forti_siem/mapping.py index 64c9f075..4fed2dbe 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/mapping.py +++ b/uncoder-core/app/translator/platforms/forti_siem/mapping.py @@ -6,6 +6,7 @@ LogSourceSignature, SourceMapping, ) +from app.translator.platforms.forti_siem.const import forti_siem_rule_details class FortiSiemLogSourceSignature(LogSourceSignature): @@ -57,4 +58,4 @@ def get_suitable_source_mappings(self, field_names: list[str], event_type: Optio return suitable_source_mappings -forti_siem_mappings = FortiSiemMappings(platform_dir="forti_siem") +forti_siem_rule_mappings = FortiSiemMappings(platform_dir="forti_siem", platform_details=forti_siem_rule_details) diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index 0696e2ba..18a4976e 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -36,7 +36,7 @@ SOURCES_EVENT_TYPES_CONTAINERS_MAP, forti_siem_rule_details, ) -from app.translator.platforms.forti_siem.mapping import FortiSiemMappings, forti_siem_mappings +from app.translator.platforms.forti_siem.mapping import FortiSiemMappings, forti_siem_rule_mappings from app.translator.platforms.forti_siem.str_value_manager import forti_siem_str_value_manager from app.translator.tools.utils import concatenate_str @@ -185,7 +185,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 @render_manager.register class FortiSiemRuleRender(PlatformQueryRender): details: PlatformDetails = forti_siem_rule_details - mappings: FortiSiemMappings = forti_siem_mappings + mappings: FortiSiemMappings = forti_siem_rule_mappings or_token = "OR" and_token = "AND" @@ -246,11 +246,14 @@ def __replace_not_tokens(self, tokens: list[QUERY_TOKEN_TYPE]) -> list[QUERY_TOK def _generate_from_tokenized_query_container_by_source_mapping( self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping ) -> str: + unmapped_fields = self.mappings.check_fields_mapping_existence( + query_container.meta_info.query_fields, source_mapping + ) is_event_type_set = False field_values = [token for token in query_container.tokens if isinstance(token, FieldValue)] mapped_fields_set = set() for field_value in field_values: - mapped_fields = self.map_field(field_value.field, source_mapping) + mapped_fields = self.mappings.map_field(field_value.field, source_mapping) mapped_fields_set = mapped_fields_set.union(set(mapped_fields)) if _EVENT_TYPE_FIELD in mapped_fields: is_event_type_set = True @@ -266,6 +269,7 @@ def _generate_from_tokenized_query_container_by_source_mapping( query=result, functions=rendered_functions.rendered, not_supported_functions=not_supported_functions, + unmapped_fields=unmapped_fields, meta_info=query_container.meta_info, source_mapping=source_mapping, fields=mapped_fields_set, @@ -299,6 +303,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, fields: Optional[set[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 @@ -316,6 +321,7 @@ def finalize_query( rule = rule.replace("", query) rule = rule.replace("", ", ".join(args_list)) rule = rule.replace("", self.get_attr_str(fields.copy())) + rule = self.wrap_with_unmapped_fields(rule, unmapped_fields) return self.wrap_with_not_supported_functions(rule, not_supported_functions) @staticmethod diff --git a/uncoder-core/app/translator/platforms/graylog/const.py b/uncoder-core/app/translator/platforms/graylog/const.py index c68bfda6..f13757f5 100644 --- a/uncoder-core/app/translator/platforms/graylog/const.py +++ b/uncoder-core/app/translator/platforms/graylog/const.py @@ -9,4 +9,4 @@ } -graylog_details = PlatformDetails(**GRAYLOG_QUERY_DETAILS) +graylog_query_details = PlatformDetails(**GRAYLOG_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/graylog/mapping.py b/uncoder-core/app/translator/platforms/graylog/mapping.py index 12e95bb3..42edc609 100644 --- a/uncoder-core/app/translator/platforms/graylog/mapping.py +++ b/uncoder-core/app/translator/platforms/graylog/mapping.py @@ -1,8 +1,4 @@ from app.translator.platforms.base.lucene.mapping import LuceneMappings +from app.translator.platforms.graylog.const import graylog_query_details - -class GraylogMappings(LuceneMappings): - pass - - -graylog_mappings = GraylogMappings(platform_dir="graylog") +graylog_query_mappings = LuceneMappings(platform_dir="graylog", platform_details=graylog_query_details) diff --git a/uncoder-core/app/translator/platforms/graylog/parsers/graylog.py b/uncoder-core/app/translator/platforms/graylog/parsers/graylog.py index a4707a09..6252cd66 100644 --- a/uncoder-core/app/translator/platforms/graylog/parsers/graylog.py +++ b/uncoder-core/app/translator/platforms/graylog/parsers/graylog.py @@ -18,12 +18,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.parsers.lucene import LuceneQueryParser -from app.translator.platforms.graylog.const import graylog_details -from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings +from app.translator.platforms.graylog.const import graylog_query_details +from app.translator.platforms.graylog.mapping import graylog_query_mappings @parser_manager.register_supported_by_roota class GraylogQueryParser(LuceneQueryParser): - details: PlatformDetails = graylog_details - mappings: GraylogMappings = graylog_mappings + details: PlatformDetails = graylog_query_details + mappings: LuceneMappings = graylog_query_mappings diff --git a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py index 986ddd93..77be5c30 100644 --- a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py +++ b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py @@ -19,19 +19,20 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender -from app.translator.platforms.graylog.const import graylog_details -from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings +from app.translator.platforms.graylog.const import graylog_query_details +from app.translator.platforms.graylog.mapping import graylog_query_mappings class GraylogFieldValue(LuceneFieldValueRender): - details: PlatformDetails = graylog_details + details: PlatformDetails = graylog_query_details @render_manager.register class GraylogQueryRender(LuceneQueryRender): - details: PlatformDetails = graylog_details - mappings: GraylogMappings = graylog_mappings + details: PlatformDetails = graylog_query_details + mappings: LuceneMappings = graylog_query_mappings or_token = "OR" field_value_render = GraylogFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/hunters/const.py b/uncoder-core/app/translator/platforms/hunters/const.py index fbeff6a1..eb61c622 100644 --- a/uncoder-core/app/translator/platforms/hunters/const.py +++ b/uncoder-core/app/translator/platforms/hunters/const.py @@ -8,4 +8,4 @@ "group_id": "hunters", } -hunters_details = PlatformDetails(**HUNTERS_QUERY_DETAILS) +hunters_query_details = PlatformDetails(**HUNTERS_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/hunters/mapping.py b/uncoder-core/app/translator/platforms/hunters/mapping.py index 28f37e28..a7236eec 100644 --- a/uncoder-core/app/translator/platforms/hunters/mapping.py +++ b/uncoder-core/app/translator/platforms/hunters/mapping.py @@ -1,4 +1,5 @@ from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.hunters.const import hunters_query_details class HuntersLogSourceSignature(LogSourceSignature): @@ -32,4 +33,4 @@ def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMap return suitable_source_mappings -hunters_mappings = HuntersMappings(platform_dir="hunters") +hunters_query_mappings = HuntersMappings(platform_dir="hunters", platform_details=hunters_query_details) diff --git a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py index 3c73c234..4e977a16 100644 --- a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py +++ b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py @@ -20,18 +20,18 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender -from app.translator.platforms.hunters.const import hunters_details -from app.translator.platforms.hunters.mapping import HuntersMappings, hunters_mappings +from app.translator.platforms.hunters.const import hunters_query_details +from app.translator.platforms.hunters.mapping import HuntersMappings, hunters_query_mappings class HuntersFieldValueRender(SqlFieldValueRender): - details: PlatformDetails = hunters_details + details: PlatformDetails = hunters_query_details @render_manager.register class HuntersQueryRender(SqlQueryRender): - details: PlatformDetails = hunters_details - mappings: HuntersMappings = hunters_mappings + details: PlatformDetails = hunters_query_details + mappings: HuntersMappings = hunters_query_mappings or_token = "OR" diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py b/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py index 477d5e29..f034c40f 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.logrhythm_axon.const import logrhythm_axon_query_details, logrhythm_axon_rule_details class LogRhythmAxonLogSourceSignature(LogSourceSignature): @@ -15,6 +16,8 @@ def __str__(self) -> str: class LogRhythmAxonMappings(BasePlatformMappings): + is_strict_mapping = True + def prepare_mapping(self) -> dict[str, SourceMapping]: source_mappings = {} for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): @@ -44,4 +47,9 @@ def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMap return suitable_source_mappings -logrhythm_axon_mappings = LogRhythmAxonMappings(platform_dir="logrhythm_axon") +logrhythm_axon_query_mappings = LogRhythmAxonMappings( + platform_dir="logrhythm_axon", platform_details=logrhythm_axon_query_details +) +logrhythm_axon_rule_mappings = LogRhythmAxonMappings( + platform_dir="logrhythm_axon", platform_details=logrhythm_axon_rule_details +) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index a38b8a64..b81f5453 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -33,7 +33,7 @@ from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_axon_query_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_query_escape_manager -from app.translator.platforms.logrhythm_axon.mapping import LogRhythmAxonMappings, logrhythm_axon_mappings +from app.translator.platforms.logrhythm_axon.mapping import LogRhythmAxonMappings, logrhythm_axon_query_mappings class LogRhythmRegexRenderException(BaseRenderException): @@ -205,10 +205,9 @@ class LogRhythmAxonQueryRender(PlatformQueryRender): field_value_render = LogRhythmAxonFieldValueRender(or_token=or_token) - mappings: LogRhythmAxonMappings = logrhythm_axon_mappings + mappings: LogRhythmAxonMappings = logrhythm_axon_query_mappings comment_symbol = "//" is_single_line_comment = True - is_strict_mapping = True @staticmethod def _finalize_search_query(query: str) -> str: @@ -220,7 +219,7 @@ def generate_prefix(self, log_source_signature: LogSourceSignature, functions_pr def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue) and token.field: try: - mapped_fields = self.map_field(token.field, source_mapping) + mapped_fields = self.mappings.map_field(token.field, source_mapping) except StrictPlatformException: try: return self.field_value_render.apply_field_value( @@ -244,6 +243,9 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> def _generate_from_tokenized_query_container_by_source_mapping( self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping ) -> str: + unmapped_fields = self.mappings.check_fields_mapping_existence( + query_container.meta_info.query_fields, source_mapping + ) prefix = self.generate_prefix(source_mapping.log_source_signature) if "product" in query_container.meta_info.parsed_logsources: prefix = f"{prefix} CONTAINS {query_container.meta_info.parsed_logsources['product'][0]}" @@ -258,6 +260,7 @@ def _generate_from_tokenized_query_container_by_source_mapping( query=result, functions=rendered_functions.rendered, not_supported_functions=not_supported_functions, + unmapped_fields=unmapped_fields, meta_info=query_container.meta_info, source_mapping=source_mapping, ) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py index 2e68c2d1..614df7d2 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py @@ -28,6 +28,7 @@ from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager +from app.translator.platforms.logrhythm_axon.mapping import LogRhythmAxonMappings, logrhythm_axon_rule_mappings from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import ( LogRhythmAxonFieldValueRender, LogRhythmAxonQueryRender, @@ -52,6 +53,7 @@ class LogRhythmAxonRuleFieldValueRender(LogRhythmAxonFieldValueRender): @render_manager.register class LogRhythmAxonRuleRender(LogRhythmAxonQueryRender): details: PlatformDetails = logrhythm_axon_rule_details + mappings: LogRhythmAxonMappings = logrhythm_axon_rule_mappings or_token = "or" field_value_render = LogRhythmAxonRuleFieldValueRender(or_token=or_token) @@ -63,6 +65,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -89,8 +92,9 @@ def finalize_query( ) if meta_info.output_table_fields: rule["observationPipeline"]["pattern"]["operations"][0]["logObserved"]["groupByFields"] = [ - self.map_field(field, source_mapping)[0] for field in meta_info.output_table_fields + self.mappings.map_field(field, source_mapping)[0] for field in meta_info.output_table_fields ] json_rule = json.dumps(rule, indent=4, sort_keys=False) + json_rule = self.wrap_with_unmapped_fields(json_rule, unmapped_fields) return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/mapping.py b/uncoder-core/app/translator/platforms/logscale/mapping.py index 3856cba8..a3e9004e 100644 --- a/uncoder-core/app/translator/platforms/logscale/mapping.py +++ b/uncoder-core/app/translator/platforms/logscale/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.logscale.const import logscale_alert_details, logscale_query_details class LogScaleLogSourceSignature(LogSourceSignature): @@ -34,4 +35,5 @@ def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMap return suitable_source_mappings -logscale_mappings = LogScaleMappings(platform_dir="logscale") +logscale_query_mappings = LogScaleMappings(platform_dir="logscale", platform_details=logscale_query_details) +logscale_alert_mappings = LogScaleMappings(platform_dir="logscale", platform_details=logscale_alert_details) diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py index 668796ae..4f6fb9d9 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py @@ -23,7 +23,7 @@ from app.translator.managers import parser_manager from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.functions import LogScaleFunctions, log_scale_functions -from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings +from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_query_mappings from app.translator.platforms.logscale.tokenizer import LogScaleTokenizer @@ -32,7 +32,7 @@ class LogScaleQueryParser(PlatformQueryParser): details: PlatformDetails = logscale_query_details platform_functions: LogScaleFunctions = log_scale_functions tokenizer = LogScaleTokenizer() - mappings: LogScaleMappings = logscale_mappings + mappings: LogScaleMappings = logscale_query_mappings wrapped_with_comment_pattern = r"^\s*/\*(?:|\n|.)*\*/" diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py index a9cbd603..d4935a4e 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py @@ -21,12 +21,14 @@ from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager from app.translator.platforms.logscale.const import logscale_alert_details +from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_alert_mappings from app.translator.platforms.logscale.parsers.logscale import LogScaleQueryParser @parser_manager.register class LogScaleAlertParser(LogScaleQueryParser, JsonRuleMixin): details: PlatformDetails = logscale_alert_details + mappings: LogScaleMappings = logscale_alert_mappings def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: rule = self.load_rule(text=text) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py index e1ed4818..1ca23243 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py @@ -17,18 +17,16 @@ ----------------------------------------------------------------- """ -from typing import Optional, Union +from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.mapping import SourceMapping from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.models.query_container import MetaInfoContainer from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.escape_manager import logscale_escape_manager from app.translator.platforms.logscale.functions import LogScaleFunctions, log_scale_functions -from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings +from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_query_mappings class LogScaleFieldValueRender(BaseFieldValueRender): @@ -95,7 +93,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @render_manager.register class LogScaleQueryRender(PlatformQueryRender): details: PlatformDetails = logscale_query_details - mappings: LogScaleMappings = logscale_mappings + mappings: LogScaleMappings = logscale_query_mappings platform_functions: LogScaleFunctions = None or_token = "or" @@ -110,18 +108,3 @@ def init_platform_functions(self) -> None: def wrap_with_comment(self, value: str) -> str: return f"/* {value} */" - - def finalize_query( - self, - prefix: str, - query: str, - functions: str, - meta_info: Optional[MetaInfoContainer] = None, - source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 - not_supported_functions: Optional[list] = None, - *args, # noqa: ARG002 - **kwargs, # noqa: ARG002 - ) -> str: - query = super().finalize_query(prefix=prefix, query=query, functions=functions) - query = self.wrap_with_meta_info(query, meta_info) - return self.wrap_with_not_supported_functions(query, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py index a6628045..57fe1edf 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py @@ -26,6 +26,7 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details +from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_alert_mappings from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValueRender, LogScaleQueryRender from app.translator.tools.utils import get_rule_description_str @@ -39,6 +40,7 @@ class LogScaleAlertFieldValueRender(LogScaleFieldValueRender): @render_manager.register class LogScaleAlertRender(LogScaleQueryRender): details: PlatformDetails = logscale_alert_details + mappings: LogScaleMappings = logscale_alert_mappings or_token = "or" field_value_render = LogScaleAlertFieldValueRender(or_token=or_token) @@ -50,6 +52,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -71,4 +74,5 @@ def finalize_query( ) rule_str = json.dumps(rule, indent=4, sort_keys=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/microsoft/const.py b/uncoder-core/app/translator/platforms/microsoft/const.py index 44dcf698..02a2a7d0 100644 --- a/uncoder-core/app/translator/platforms/microsoft/const.py +++ b/uncoder-core/app/translator/platforms/microsoft/const.py @@ -42,6 +42,6 @@ "group_id": "microsoft-defender", } -microsoft_defender_details = PlatformDetails(**MICROSOFT_DEFENDER_DETAILS) +microsoft_defender_query_details = PlatformDetails(**MICROSOFT_DEFENDER_DETAILS) microsoft_sentinel_query_details = PlatformDetails(**MICROSOFT_SENTINEL_QUERY_DETAILS) microsoft_sentinel_rule_details = PlatformDetails(**MICROSOFT_SENTINEL_RULE_DETAILS) diff --git a/uncoder-core/app/translator/platforms/microsoft/mapping.py b/uncoder-core/app/translator/platforms/microsoft/mapping.py index 0c32b522..4add9858 100644 --- a/uncoder-core/app/translator/platforms/microsoft/mapping.py +++ b/uncoder-core/app/translator/platforms/microsoft/mapping.py @@ -1,6 +1,11 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.microsoft.const import ( + microsoft_defender_query_details, + microsoft_sentinel_query_details, + microsoft_sentinel_rule_details, +) class MicrosoftSentinelLogSourceSignature(LogSourceSignature): @@ -37,7 +42,12 @@ def get_suitable_source_mappings(self, field_names: list[str], table: list[str]) return suitable_source_mappings -microsoft_sentinel_mappings = MicrosoftSentinelMappings(platform_dir="microsoft_sentinel") +microsoft_sentinel_query_mappings = MicrosoftSentinelMappings( + platform_dir="microsoft_sentinel", platform_details=microsoft_sentinel_query_details +) +microsoft_sentinel_rule_mappings = MicrosoftSentinelMappings( + platform_dir="microsoft_sentinel", platform_details=microsoft_sentinel_rule_details +) class MicrosoftDefenderLogSourceSignature(MicrosoftSentinelLogSourceSignature): @@ -45,10 +55,14 @@ class MicrosoftDefenderLogSourceSignature(MicrosoftSentinelLogSourceSignature): class MicrosoftDefenderMappings(MicrosoftSentinelMappings): + is_strict_mapping = True + def prepare_log_source_signature(self, mapping: dict) -> MicrosoftDefenderLogSourceSignature: tables = mapping.get("log_source", {}).get("table") default_log_source = mapping["default_log_source"] return MicrosoftDefenderLogSourceSignature(tables=tables, default_source=default_log_source) -microsoft_defender_mappings = MicrosoftDefenderMappings(platform_dir="microsoft_defender") +microsoft_defender_query_mappings = MicrosoftDefenderMappings( + platform_dir="microsoft_defender", platform_details=microsoft_defender_query_details +) diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_defender.py index a903f0b3..99fc551d 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_defender.py @@ -18,14 +18,14 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager -from app.translator.platforms.microsoft.const import microsoft_defender_details +from app.translator.platforms.microsoft.const import microsoft_defender_query_details from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions -from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings +from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_query_mappings from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser @parser_manager.register_supported_by_roota class MicrosoftDefenderQueryParser(MicrosoftSentinelQueryParser): - mappings: MicrosoftDefenderMappings = microsoft_defender_mappings - details: PlatformDetails = microsoft_defender_details + mappings: MicrosoftDefenderMappings = microsoft_defender_query_mappings + details: PlatformDetails = microsoft_defender_query_details platform_functions: MicrosoftFunctions = microsoft_defender_functions diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index 2325367f..24d522e9 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -23,14 +23,14 @@ from app.translator.managers import parser_manager from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_sentinel_functions -from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings +from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_query_mappings from app.translator.platforms.microsoft.tokenizer import MicrosoftSentinelTokenizer @parser_manager.register_supported_by_roota class MicrosoftSentinelQueryParser(PlatformQueryParser): platform_functions: MicrosoftFunctions = microsoft_sentinel_functions - mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings + mappings: MicrosoftSentinelMappings = microsoft_sentinel_query_mappings tokenizer = MicrosoftSentinelTokenizer() details: PlatformDetails = microsoft_sentinel_query_details diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py index 9cf400e2..ab60a21f 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py @@ -21,12 +21,14 @@ from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager from app.translator.platforms.microsoft.const import microsoft_sentinel_rule_details +from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_rule_mappings from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser @parser_manager.register class MicrosoftSentinelRuleParser(MicrosoftSentinelQueryParser, JsonRuleMixin): details: PlatformDetails = microsoft_sentinel_rule_details + mappings: MicrosoftSentinelMappings = microsoft_sentinel_rule_mappings def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: rule = self.load_rule(text=text) diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py index 38617b55..69953044 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py @@ -19,9 +19,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.microsoft.const import microsoft_defender_details +from app.translator.platforms.microsoft.const import microsoft_defender_query_details from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions -from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings +from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_query_mappings from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, @@ -29,19 +29,17 @@ class MicrosoftDefenderFieldValueRender(MicrosoftSentinelFieldValueRender): - details: PlatformDetails = microsoft_defender_details + details: PlatformDetails = microsoft_defender_query_details @render_manager.register class MicrosoftDefenderQueryRender(MicrosoftSentinelQueryRender): - mappings: MicrosoftDefenderMappings = microsoft_defender_mappings - details: PlatformDetails = microsoft_defender_details + mappings: MicrosoftDefenderMappings = microsoft_defender_query_mappings + details: PlatformDetails = microsoft_defender_query_details platform_functions: MicrosoftFunctions = None or_token = "or" field_value_render = MicrosoftDefenderFieldValueRender(or_token=or_token) - is_strict_mapping = True - def init_platform_functions(self) -> None: self.platform_functions = microsoft_defender_functions self.platform_functions.platform_query_render = self diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py index 621decb1..72521800 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py @@ -22,13 +22,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render_cti import RenderCTI from app.translator.managers import render_cti_manager -from app.translator.platforms.microsoft.const import microsoft_defender_details +from app.translator.platforms.microsoft.const import microsoft_defender_query_details from app.translator.platforms.microsoft.mappings.mdatp_cti import DEFAULT_MICROSOFT_DEFENDER_MAPPING @render_cti_manager.register class MicrosoftDefenderCTI(RenderCTI): - details: PlatformDetails = microsoft_defender_details + details: PlatformDetails = microsoft_defender_query_details field_value_templates_map: ClassVar[dict[str, str]] = { "default": '{key} =~ "{value}"', diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py index 7ef6f1f9..961fe98a 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py @@ -27,7 +27,7 @@ from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.escape_manager import microsoft_escape_manager from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_sentinel_functions -from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings +from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_query_mappings class MicrosoftSentinelFieldValueRender(BaseFieldValueRender): @@ -130,7 +130,7 @@ class MicrosoftSentinelQueryRender(PlatformQueryRender): field_value_render = MicrosoftSentinelFieldValueRender(or_token=or_token) - mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings + mappings: MicrosoftSentinelMappings = microsoft_sentinel_query_mappings comment_symbol = "//" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index b5631ef5..1a64f14b 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -27,6 +27,7 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details +from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_rule_mappings from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, @@ -49,6 +50,7 @@ class MicrosoftSentinelRuleFieldValueRender(MicrosoftSentinelFieldValueRender): @render_manager.register class MicrosoftSentinelRuleRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_sentinel_rule_details + mappings: MicrosoftSentinelMappings = microsoft_sentinel_rule_mappings or_token = "or" field_value_render = MicrosoftSentinelRuleFieldValueRender(or_token=or_token) @@ -75,6 +77,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -92,4 +95,5 @@ def finalize_query( rule["tactics"] = mitre_tactics rule["techniques"] = mitre_techniques json_rule = json.dumps(rule, indent=4, sort_keys=False) + json_rule = self.wrap_with_unmapped_fields(json_rule, unmapped_fields) return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/opensearch/mapping.py b/uncoder-core/app/translator/platforms/opensearch/mapping.py index 57b4190d..ad0222ed 100644 --- a/uncoder-core/app/translator/platforms/opensearch/mapping.py +++ b/uncoder-core/app/translator/platforms/opensearch/mapping.py @@ -1,8 +1,5 @@ from app.translator.platforms.base.lucene.mapping import LuceneMappings +from app.translator.platforms.opensearch.const import opensearch_query_details, opensearch_rule_details - -class OpenSearchMappings(LuceneMappings): - pass - - -opensearch_mappings = OpenSearchMappings(platform_dir="opensearch") +opensearch_query_mappings = LuceneMappings(platform_dir="opensearch", platform_details=opensearch_query_details) +opensearch_rule_mappings = LuceneMappings(platform_dir="opensearch", platform_details=opensearch_rule_details) diff --git a/uncoder-core/app/translator/platforms/opensearch/parsers/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/parsers/opensearch.py index b07e01f1..6a3a4444 100644 --- a/uncoder-core/app/translator/platforms/opensearch/parsers/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/parsers/opensearch.py @@ -18,12 +18,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.parsers.lucene import LuceneQueryParser from app.translator.platforms.opensearch.const import opensearch_query_details -from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings +from app.translator.platforms.opensearch.mapping import opensearch_query_mappings @parser_manager.register_supported_by_roota class OpenSearchQueryParser(LuceneQueryParser): details: PlatformDetails = opensearch_query_details - mappings: OpenSearchMappings = opensearch_mappings + mappings: LuceneMappings = opensearch_query_mappings diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py index 3298c106..a1a3f1a6 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py @@ -24,9 +24,10 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.opensearch.const import opensearch_query_details -from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings +from app.translator.platforms.opensearch.mapping import opensearch_query_mappings class OpenSearchFieldValueRender(LuceneFieldValueRender): @@ -99,7 +100,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @render_manager.register class OpenSearchQueryRender(LuceneQueryRender): details: PlatformDetails = opensearch_query_details - mappings: OpenSearchMappings = opensearch_mappings + mappings: LuceneMappings = opensearch_query_mappings or_token = "OR" field_value_render = OpenSearchFieldValueRender(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py index c5c67ed4..ac5f74fc 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -28,8 +28,9 @@ from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.opensearch.const import OPENSEARCH_RULE, opensearch_rule_details -from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings +from app.translator.platforms.opensearch.mapping import opensearch_rule_mappings from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValueRender, OpenSearchQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated AWS OpenSearch Rule" @@ -43,7 +44,7 @@ class OpenSearchRuleFieldValueRender(OpenSearchFieldValueRender): @render_manager.register class OpenSearchRuleRender(OpenSearchQueryRender): details: PlatformDetails = opensearch_rule_details - mappings: OpenSearchMappings = opensearch_mappings + mappings: LuceneMappings = opensearch_rule_mappings or_token = "OR" and_token = "AND" @@ -63,6 +64,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -76,11 +78,12 @@ def finalize_query( rule["triggers"][0]["severity"] = _SEVERITIES_MAP[meta_info.severity] rule["triggers"][0]["actions"][0]["message_template"]["source"] = str(source).replace(", ", ",\n") rule_str = json.dumps(rule, indent=4, sort_keys=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue) and token.field: - for field in self.map_field(token.field, source_mapping): + for field in self.mappings.map_field(token.field, source_mapping): self.fields.update({field: f"{{ctx.results.0.hits.hits.0._source.{field}}}"}) return super().apply_token(token, source_mapping) diff --git a/uncoder-core/app/translator/platforms/palo_alto/mapping.py b/uncoder-core/app/translator/platforms/palo_alto/mapping.py index fc6a7797..3dd5e4c9 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/mapping.py +++ b/uncoder-core/app/translator/platforms/palo_alto/mapping.py @@ -7,6 +7,7 @@ LogSourceSignature, SourceMapping, ) +from app.translator.platforms.palo_alto.const import cortex_xql_query_details class CortexXQLLogSourceSignature(LogSourceSignature): @@ -73,4 +74,6 @@ def get_suitable_source_mappings( return suitable_source_mappings -cortex_xql_mappings = CortexXQLMappings(platform_dir="palo_alto_cortex") +cortex_xql_query_mappings = CortexXQLMappings( + platform_dir="palo_alto_cortex", platform_details=cortex_xql_query_details +) diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index f2a74bc8..6984b412 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -16,18 +16,15 @@ limitations under the License. ----------------------------------------------------------------- """ -from contextlib import suppress from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.const import QUERY_TOKEN_TYPE -from app.translator.core.context_vars import preset_log_source_str_ctx_var, return_only_first_query_ctx_var +from app.translator.core.context_vars import preset_log_source_str_ctx_var from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType -from app.translator.core.exceptions.core import StrictPlatformException -from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping +from app.translator.core.mapping import SourceMapping from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.models.query_container import TokenizedQueryContainer from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue @@ -37,7 +34,7 @@ from app.translator.platforms.palo_alto.mapping import ( CortexXQLLogSourceSignature, CortexXQLMappings, - cortex_xql_mappings, + cortex_xql_query_mappings, ) from app.translator.platforms.palo_alto.str_value_manager import cortex_xql_str_value_manager @@ -73,7 +70,8 @@ def _wrap_str_value(value: str) -> str: def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): values = ", ".join( - f"{self._pre_process_value(field, str(v), value_type=ValueType.value, wrap_str=True)}" for v in value + f"{self._pre_process_value(field, str(v) if isinstance(v, int) else v, ValueType.value, True)}" + for v in value ) return f"{field} in ({values})" @@ -167,8 +165,7 @@ class CortexXQLFieldFieldRender(BaseFieldFieldRender): @render_manager.register class CortexXQLQueryRender(PlatformQueryRender): details: PlatformDetails = cortex_xql_query_details - mappings: CortexXQLMappings = cortex_xql_mappings - is_strict_mapping = True + mappings: CortexXQLMappings = cortex_xql_query_mappings predefined_fields_map = PREDEFINED_FIELDS_MAP raw_log_field_patterns_map: ClassVar[dict[str, str]] = { "regex": '| alter {field} = regextract(to_json_string(action_evtlog_data_fields)->{field}{{}}, "\\"(.*)\\"")', @@ -224,32 +221,3 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> @staticmethod def _finalize_search_query(query: str) -> str: return f"| filter {query}" if query else "" - - def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: - queries_map = {} - errors = [] - source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) - - last_mapping_index = len(source_mappings) - 1 - for index, source_mapping in enumerate(source_mappings): - try: - finalized_query = self._generate_from_tokenized_query_container_by_source_mapping( - query_container, source_mapping - ) - if return_only_first_query_ctx_var.get() is True: - return finalized_query - queries_map[source_mapping.source_id] = finalized_query - except StrictPlatformException as err: - errors.append(err) - if index != last_mapping_index or source_mapping.source_id == DEFAULT_MAPPING_NAME or queries_map: - continue - - with suppress(StrictPlatformException): - finalized_query = self._generate_from_tokenized_query_container_by_source_mapping( - query_container, self.mappings.get_source_mapping(DEFAULT_MAPPING_NAME) - ) - queries_map[source_mapping.source_id] = finalized_query - - if not queries_map and errors: - raise errors[0] - return self.finalize(queries_map) diff --git a/uncoder-core/app/translator/platforms/qradar/mapping.py b/uncoder-core/app/translator/platforms/qradar/mapping.py new file mode 100644 index 00000000..e179e73b --- /dev/null +++ b/uncoder-core/app/translator/platforms/qradar/mapping.py @@ -0,0 +1,4 @@ +from app.translator.platforms.base.aql.mapping import AQLMappings +from app.translator.platforms.qradar.const import qradar_query_details + +qradar_query_mappings = AQLMappings(platform_dir="qradar", platform_details=qradar_query_details) diff --git a/uncoder-core/app/translator/platforms/qradar/parsers/qradar.py b/uncoder-core/app/translator/platforms/qradar/parsers/qradar.py index c74d3f1f..ddb2f0cb 100644 --- a/uncoder-core/app/translator/platforms/qradar/parsers/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/parsers/qradar.py @@ -18,12 +18,15 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager +from app.translator.platforms.base.aql.mapping import AQLMappings from app.translator.platforms.base.aql.parsers.aql import AQLQueryParser from app.translator.platforms.qradar.const import qradar_query_details +from app.translator.platforms.qradar.mapping import qradar_query_mappings @parser_manager.register_supported_by_roota class QradarQueryParser(AQLQueryParser): details: PlatformDetails = qradar_query_details + mappings: AQLMappings = qradar_query_mappings wrapped_with_comment_pattern = r"^\s*/\*(?:|\n|.)*\*/" diff --git a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py index cf4a7d51..a6846dad 100644 --- a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py @@ -19,8 +19,10 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager +from app.translator.platforms.base.aql.mapping import AQLMappings from app.translator.platforms.base.aql.renders.aql import AQLFieldValueRender, AQLQueryRender from app.translator.platforms.qradar.const import qradar_query_details +from app.translator.platforms.qradar.mapping import qradar_query_mappings class QradarFieldValueRender(AQLFieldValueRender): @@ -30,4 +32,5 @@ class QradarFieldValueRender(AQLFieldValueRender): @render_manager.register class QradarQueryRender(AQLQueryRender): details: PlatformDetails = qradar_query_details + mappings: AQLMappings = qradar_query_mappings field_value_render = QradarFieldValueRender(or_token=AQLQueryRender.or_token) diff --git a/uncoder-core/app/translator/platforms/sigma/const.py b/uncoder-core/app/translator/platforms/sigma/const.py index b7f88a98..aaedda41 100644 --- a/uncoder-core/app/translator/platforms/sigma/const.py +++ b/uncoder-core/app/translator/platforms/sigma/const.py @@ -1,3 +1,5 @@ +from app.translator.core.models.platform_details import PlatformDetails + SIGMA_RULE_DETAILS = { "name": "Sigma", "platform_id": "sigma", @@ -5,3 +7,5 @@ "group_name": "Sigma", "group_id": "sigma", } + +sigma_rule_details = PlatformDetails(**SIGMA_RULE_DETAILS) diff --git a/uncoder-core/app/translator/platforms/sigma/mapping.py b/uncoder-core/app/translator/platforms/sigma/mapping.py index 1af791ac..769e5c25 100644 --- a/uncoder-core/app/translator/platforms/sigma/mapping.py +++ b/uncoder-core/app/translator/platforms/sigma/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.sigma.const import sigma_rule_details class SigmaLogSourceSignature(LogSourceSignature): @@ -59,4 +60,4 @@ def get_suitable_source_mappings( return suitable_source_mappings or [self._source_mappings[DEFAULT_MAPPING_NAME]] -sigma_mappings = SigmaMappings(platform_dir="sigma") +sigma_rule_mappings = SigmaMappings(platform_dir="sigma", platform_details=sigma_rule_details) diff --git a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py index 5dd16651..65ebc822 100644 --- a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py @@ -28,17 +28,17 @@ from app.translator.core.parser import QueryParser from app.translator.core.tokenizer import QueryTokenizer from app.translator.managers import parser_manager -from app.translator.platforms.sigma.const import SIGMA_RULE_DETAILS -from app.translator.platforms.sigma.mapping import SigmaMappings, sigma_mappings +from app.translator.platforms.sigma.const import sigma_rule_details +from app.translator.platforms.sigma.mapping import SigmaMappings, sigma_rule_mappings from app.translator.platforms.sigma.tokenizer import SigmaConditionTokenizer, SigmaTokenizer @parser_manager.register_main class SigmaParser(QueryParser, YamlRuleMixin): - details: PlatformDetails = PlatformDetails(**SIGMA_RULE_DETAILS) + details: PlatformDetails = sigma_rule_details condition_tokenizer = SigmaConditionTokenizer() tokenizer: SigmaTokenizer = SigmaTokenizer() - mappings: SigmaMappings = sigma_mappings + mappings: SigmaMappings = sigma_rule_mappings mandatory_fields = {"title", "description", "logsource", "detection"} wrapped_with_comment_pattern = r"^\s*#.*(?:\n|$)" diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index 9eaae45c..25494e7f 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -31,8 +31,8 @@ from app.translator.core.render import QueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager -from app.translator.platforms.sigma.const import SIGMA_RULE_DETAILS -from app.translator.platforms.sigma.mapping import SigmaLogSourceSignature, SigmaMappings, sigma_mappings +from app.translator.platforms.sigma.const import sigma_rule_details +from app.translator.platforms.sigma.mapping import SigmaLogSourceSignature, SigmaMappings, sigma_rule_mappings from app.translator.platforms.sigma.models.compiler import DataStructureCompiler from app.translator.platforms.sigma.models.group import Group from app.translator.platforms.sigma.models.operator import AND, NOT, OR @@ -51,8 +51,8 @@ class SigmaRender(QueryRender): comment_symbol = "#" is_single_line_comment = True - mappings: SigmaMappings = sigma_mappings - details: PlatformDetails = PlatformDetails(**SIGMA_RULE_DETAILS) + mappings: SigmaMappings = sigma_rule_mappings + details: PlatformDetails = sigma_rule_details str_value_manager = sigma_str_value_manager @property @@ -198,15 +198,8 @@ def generate_not(self, data: Any, source_mapping: SourceMapping): not_node["condition"] = f"not {condition}" return not_node - @staticmethod - def map_field(source_mapping: SourceMapping, generic_field_name: str) -> str: - field_name = source_mapping.fields_mapping.get_platform_field_name(generic_field_name) - return field_name or generic_field_name - def generate_field(self, data: FieldValue, source_mapping: SourceMapping): - source_id = source_mapping.source_id - generic_field_name = data.field.get_generic_field_name(source_id) or data.field.source_name - field_name = self.map_field(source_mapping, generic_field_name) + field_name = self.mappings.map_field(data.field, source_mapping)[0] if data.operator.token_type not in ( OperatorType.EQ, OperatorType.LT, diff --git a/uncoder-core/app/translator/platforms/splunk/mapping.py b/uncoder-core/app/translator/platforms/splunk/mapping.py index 1851b8af..5559a947 100644 --- a/uncoder-core/app/translator/platforms/splunk/mapping.py +++ b/uncoder-core/app/translator/platforms/splunk/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.splunk.const import splunk_alert_details, splunk_query_details class SplunkLogSourceSignature(LogSourceSignature): @@ -69,4 +70,5 @@ def get_suitable_source_mappings( return suitable_source_mappings or [self._source_mappings[DEFAULT_MAPPING_NAME]] -splunk_mappings = SplunkMappings(platform_dir="splunk") +splunk_query_mappings = SplunkMappings(platform_dir="splunk", platform_details=splunk_query_details) +splunk_alert_mappings = SplunkMappings(platform_dir="splunk", platform_details=splunk_alert_details) diff --git a/uncoder-core/app/translator/platforms/splunk/parsers/splunk.py b/uncoder-core/app/translator/platforms/splunk/parsers/splunk.py index e1030b55..2370717a 100644 --- a/uncoder-core/app/translator/platforms/splunk/parsers/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/parsers/splunk.py @@ -21,15 +21,14 @@ from app.translator.platforms.base.spl.parsers.spl import SplQueryParser from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions -from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings +from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_query_mappings @parser_manager.register_supported_by_roota class SplunkQueryParser(SplQueryParser): details: PlatformDetails = splunk_query_details + mappings: SplunkMappings = splunk_query_mappings + platform_functions: SplunkFunctions = splunk_functions log_source_pattern = r"^___source_type___\s*=\s*(?:\"(?P[%a-zA-Z_*:0-9\-/]+)\"|(?P[%a-zA-Z_*:0-9\-/]+))(?:\s+(?:and|or)\s+|\s+)?" # noqa: E501 log_source_key_types = ("index", "source", "sourcetype", "sourcecategory") - - mappings: SplunkMappings = splunk_mappings - platform_functions: SplunkFunctions = splunk_functions diff --git a/uncoder-core/app/translator/platforms/splunk/parsers/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/parsers/splunk_alert.py index 1049ffbf..903478a9 100644 --- a/uncoder-core/app/translator/platforms/splunk/parsers/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/parsers/splunk_alert.py @@ -22,12 +22,14 @@ from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager from app.translator.platforms.splunk.const import splunk_alert_details +from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_alert_mappings from app.translator.platforms.splunk.parsers.splunk import SplunkQueryParser @parser_manager.register class SplunkAlertParser(SplunkQueryParser): details: PlatformDetails = splunk_alert_details + mappings: SplunkMappings = splunk_alert_mappings def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: query = re.search(r"search\s*=\s*(?P.+)", text).group("query") diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py index e14c6bfc..7a50d3d1 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py @@ -22,7 +22,7 @@ from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions -from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings +from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_query_mappings class SplunkFieldValueRender(SplFieldValueRender): @@ -32,12 +32,12 @@ class SplunkFieldValueRender(SplFieldValueRender): @render_manager.register class SplunkQueryRender(SplQueryRender): details: PlatformDetails = splunk_query_details + mappings: SplunkMappings = splunk_query_mappings + platform_functions: SplunkFunctions = None or_token = "OR" field_value_render = SplunkFieldValueRender(or_token=or_token) - mappings: SplunkMappings = splunk_mappings - platform_functions: SplunkFunctions = None def init_platform_functions(self) -> None: self.platform_functions = splunk_functions diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py index 5dc2096a..01c27525 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py @@ -25,6 +25,7 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details +from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_alert_mappings from app.translator.platforms.splunk.renders.splunk import SplunkFieldValueRender, SplunkQueryRender from app.translator.tools.utils import get_rule_description_str @@ -39,6 +40,8 @@ class SplunkAlertFieldValueRender(SplunkFieldValueRender): @render_manager.register class SplunkAlertRender(SplunkQueryRender): details: PlatformDetails = splunk_alert_details + mappings: SplunkMappings = splunk_alert_mappings + or_token = "OR" field_value_render = SplunkAlertFieldValueRender(or_token=or_token) @@ -59,6 +62,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -74,4 +78,5 @@ def finalize_query( if mitre_techniques: mitre_str = f"action.correlationsearch.annotations = {mitre_techniques})" rule = rule.replace("", mitre_str) + rule = self.wrap_with_unmapped_fields(rule, unmapped_fields) return self.wrap_with_not_supported_functions(rule, not_supported_functions) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy