From b94266a46c2c659155acb156e9a51a56806ae63b Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Mon, 24 Jun 2024 13:03:52 +0300 Subject: [PATCH 1/6] FieldField class, renders improvements --- uncoder-core/app/translator/core/const.py | 6 ++ .../app/translator/core/context_vars.py | 6 ++ .../app/translator/core/models/field.py | 16 +++ .../translator/core/models/query_container.py | 2 +- uncoder-core/app/translator/core/parser.py | 3 +- uncoder-core/app/translator/core/render.py | 97 +++++++++++-------- uncoder-core/app/translator/core/tokenizer.py | 20 +++- .../platforms/athena/renders/athena.py | 6 +- .../platforms/base/aql/renders/aql.py | 6 +- .../platforms/base/lucene/renders/lucene.py | 4 +- .../platforms/base/spl/renders/spl.py | 4 +- .../platforms/base/sql/renders/sql.py | 4 +- .../platforms/chronicle/renders/chronicle.py | 6 +- .../chronicle/renders/chronicle_rule.py | 6 +- .../crowdstrike/renders/crowdstrike.py | 6 +- .../elasticsearch/renders/detection_rule.py | 7 +- .../elasticsearch/renders/elast_alert.py | 7 +- .../elasticsearch/renders/elasticsearch.py | 6 +- .../platforms/elasticsearch/renders/kibana.py | 7 +- .../elasticsearch/renders/xpack_watcher.py | 7 +- .../forti_siem/renders/forti_siem_rule.py | 16 ++- .../platforms/graylog/renders/graylog.py | 6 +- .../platforms/hunters/renders/hunters.py | 6 +- .../renders/logrhythm_axon_query.py | 24 ++--- .../renders/logrhythm_axon_rule.py | 11 +-- .../platforms/logscale/renders/logscale.py | 13 +-- .../logscale/renders/logscale_alert.py | 13 +-- .../microsoft/renders/microsoft_defender.py | 6 +- .../microsoft/renders/microsoft_sentinel.py | 6 +- .../renders/microsoft_sentinel_rule.py | 11 +-- .../opensearch/renders/opensearch.py | 6 +- .../opensearch/renders/opensearch_rule.py | 11 +-- .../palo_alto/renders/cortex_xsiam.py | 23 ++++- .../platforms/qradar/renders/qradar.py | 5 +- .../platforms/sigma/renders/sigma.py | 8 +- .../platforms/splunk/renders/splunk.py | 6 +- .../platforms/splunk/renders/splunk_alert.py | 11 +-- 37 files changed, 221 insertions(+), 187 deletions(-) create mode 100644 uncoder-core/app/translator/core/const.py diff --git a/uncoder-core/app/translator/core/const.py b/uncoder-core/app/translator/core/const.py new file mode 100644 index 00000000..a8788ada --- /dev/null +++ b/uncoder-core/app/translator/core/const.py @@ -0,0 +1,6 @@ +from typing import Union + +from app.translator.core.models.field import Alias, Field, FieldValue, Keyword +from app.translator.core.models.identifier import Identifier + +TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias] diff --git a/uncoder-core/app/translator/core/context_vars.py b/uncoder-core/app/translator/core/context_vars.py index 591883d8..8ff6ccfb 100644 --- a/uncoder-core/app/translator/core/context_vars.py +++ b/uncoder-core/app/translator/core/context_vars.py @@ -1,4 +1,10 @@ from contextvars import ContextVar +from typing import Optional return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False) """Set to True to return only first query if rendered multiple options""" + +wrap_query_with_meta_info_ctx_var: ContextVar[bool] = ContextVar("wrap_query_with_meta_info_ctx_var", default=True) +"""Set to False not to wrap query with meta info commentary""" + +preset_log_source_str_ctx_var: ContextVar[Optional[str]] = ContextVar("preset_log_source_str_ctx_var", default=None) diff --git a/uncoder-core/app/translator/core/models/field.py b/uncoder-core/app/translator/core/models/field.py index 10b661b0..0c430382 100644 --- a/uncoder-core/app/translator/core/models/field.py +++ b/uncoder-core/app/translator/core/models/field.py @@ -37,6 +37,22 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma self.__generic_names_map = generic_names_map +class FieldField: + def __init__( + self, + source_name_left: str, + operator: Identifier, + source_name_right: str, + is_alias_left: bool = False, + is_alias_right: bool = False, + ): + self.field_left = Field(source_name=source_name_left) + self.alias_left = Alias(name=source_name_left) if is_alias_left else None + self.operator = operator + self.field_right = Field(source_name=source_name_right) + self.alias_right = Alias(name=source_name_right) if is_alias_right else None + + class FieldValue: def __init__( self, diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index dccfc180..0d90f237 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -3,11 +3,11 @@ from datetime import datetime from typing import Optional +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import DEFAULT_MAPPING_NAME from app.translator.core.models.field import Field from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.tokenizer import TOKEN_TYPE class MetaInfoContainer: diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 7cc10ec1..18b50739 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping @@ -28,7 +29,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer -from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer +from app.translator.core.tokenizer import QueryTokenizer class QueryParser(ABC): diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index bf28b4f6..636f718f 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -16,13 +16,14 @@ limitations under the License. ----------------------------------------------------------------- """ - +import itertools from abc import ABC, abstractmethod from collections.abc import Callable from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.const import TOKEN_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -30,22 +31,21 @@ from app.translator.core.exceptions.parser import UnsupportedOperatorException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function, RenderedFunctions from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.str_value_manager import StrValue, StrValueManager -from app.translator.core.tokenizer import TOKEN_TYPE -class BaseQueryFieldValue(ABC): +class BaseFieldValueRender(ABC): details: PlatformDetails = None escape_manager: EscapeManager = None str_value_manager: StrValueManager = None def __init__(self, or_token: str): - self.field_value: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { + self.modifiers_map: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { OperatorType.EQ: self.equal_modifier, OperatorType.NOT_EQ: self.not_equal_modifier, OperatorType.LT: self.less_modifier, @@ -155,11 +155,20 @@ def apply_value(self, value: Union[str, int], value_type: str = ValueType.value) return self.escape_manager.escape(value, value_type) def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VALUE_TYPE) -> str: - if modifier_function := self.field_value.get(operator.token_type): + if modifier_function := self.modifiers_map.get(operator.token_type): return modifier_function(field, value) raise UnsupportedOperatorException(operator.token_type) +class BaseFieldFieldRender(ABC): + operators_map: ClassVar[dict[str, str]] = {} + + def apply_field_field(self, field_left: str, operator: Identifier, field_right: str) -> str: + if mapped_operator := self.operators_map.get(operator.token_type): + return f"{field_left} {mapped_operator} {field_right}" + raise UnsupportedOperatorException(operator.token_type) + + class QueryRender(ABC): comment_symbol: str = None details: PlatformDetails = None @@ -180,6 +189,13 @@ def render_not_supported_functions(self, not_supported_functions: list) -> str: not_supported_functions_str = "\n".join(line_template + func.lstrip() for func in not_supported_functions) return "\n\n" + self.wrap_with_comment(f"{self.unsupported_functions_text}\n{not_supported_functions_str}") + def wrap_with_not_supported_functions(self, query: str, not_supported_functions: Optional[list] = None) -> str: + if not_supported_functions and wrap_query_with_meta_info_ctx_var.get(): + rendered_not_supported = self.render_not_supported_functions(not_supported_functions) + return query + rendered_not_supported + + return query + def wrap_with_comment(self, value: str) -> str: return f"{self.comment_symbol} {value}" @@ -199,13 +215,14 @@ class PlatformQueryRender(QueryRender): group_token = "(%s)" query_parts_delimiter = " " - field_value_map = BaseQueryFieldValue(or_token=or_token) + field_field_render = BaseFieldFieldRender() + field_value_render = BaseFieldValueRender(or_token=or_token) raw_log_field_pattern_map: ClassVar[dict[str, str]] = None def __init__(self): super().__init__() - self.operator_map = { + self.logical_operators_map = { LogicalOperatorType.AND: f" {self.and_token} ", LogicalOperatorType.OR: f" {self.or_token} ", LogicalOperatorType.NOT: f" {self.not_token} ", @@ -233,31 +250,34 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]: def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): - if token.alias: - field_name = token.alias.name - else: - mapped_fields = self.map_field(token.field, source_mapping) - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value( - field=field, operator=token.operator, value=token.value - ) - for field in mapped_fields - ] - ) - - field_name = mapped_fields[0] - - return self.field_value_map.apply_field_value(field=field_name, operator=token.operator, value=token.value) - + mapped_fields = [token.alias.name] if token.alias else self.map_field(token.field, source_mapping) + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] + ) + return self.group_token % joined if len(mapped_fields) > 1 else joined + if isinstance(token, FieldField): + alias_left, field_left = token.alias_left, token.field_left + mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping) + alias_right, field_right = token.alias_right, token.field_right + mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping) + cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right)) + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_field_render.apply_field_field(pair[0], token.operator, pair[1]) + for pair in cross_paired_fields + ] + ) + return self.group_token % joined if len(cross_paired_fields) > 1 else joined if isinstance(token, Function): func_render = self.platform_functions.manager.get_in_query_render(token.name) return func_render.render(token, source_mapping) if isinstance(token, Keyword): - return self.field_value_map.apply_field_value(field="", operator=token.operator, value=token.value) + return self.field_value_render.apply_field_value(field="", operator=token.operator, value=token.value) if token.token_type in LogicalOperatorType: - return self.operator_map.get(token.token_type) + return self.logical_operators_map.get(token.token_type) return token.token_type @@ -267,8 +287,8 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) return "".join(result_values) - def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str: - if meta_info and (meta_info.id or meta_info.title): + def wrap_with_meta_info(self, query: str, meta_info: MetaInfoContainer) -> str: + if wrap_query_with_meta_info_ctx_var.get() and meta_info and (meta_info.id or meta_info.title): meta_info_dict = { "name: ": meta_info.title, "uuid: ": meta_info.id, @@ -301,11 +321,8 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = self._join_query_parts(prefix, query, functions) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) @staticmethod def unique_queries(queries_map: dict[str, str]) -> dict[str, dict[str]]: @@ -336,7 +353,7 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp return source_mappings - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: return self.finalize_query( prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info ) @@ -374,7 +391,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap defined_raw_log_fields.append(prefix) return "\n".join(defined_raw_log_fields) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} errors = [] source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -411,6 +428,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index 45486ef1..2ecbc39c 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Any, ClassVar, Optional, Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -29,18 +30,18 @@ UnsupportedOperatorException, ) from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function from app.translator.core.models.functions.eval import EvalArg from app.translator.core.models.functions.group_by import GroupByFunction +from app.translator.core.models.functions.join import JoinFunction from app.translator.core.models.functions.rename import RenameArg from app.translator.core.models.functions.sort import SortArg +from app.translator.core.models.functions.union import UnionFunction from app.translator.core.models.identifier import Identifier from app.translator.core.str_value_manager import StrValue, StrValueManager from app.translator.tools.utils import get_match_group -TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field] - class BaseTokenizer(ABC): @abstractmethod @@ -323,13 +324,18 @@ def filter_tokens( ) -> list[TOKEN_TYPE]: return [token for token in tokens if isinstance(token, token_type)] - def get_field_tokens_from_func_args( + def get_field_tokens_from_func_args( # noqa: PLR0912 self, args: list[Union[Field, FieldValue, Keyword, Identifier, Function, SortArg]] ) -> list[Field]: result = [] for arg in args: if isinstance(arg, Field): result.append(arg) + elif isinstance(arg, FieldField): + if not arg.alias_left or arg.alias_left.name != arg.field_left.source_name: + result.append(arg.field_left) + if not arg.alias_right or arg.alias_right.name != arg.field_right.source_name: + result.append(arg.field_right) elif isinstance(arg, FieldValue): if not arg.alias or arg.alias.name != arg.field.source_name: result.append(arg.field) @@ -337,6 +343,12 @@ def get_field_tokens_from_func_args( result.extend(self.get_field_tokens_from_func_args(args=arg.args)) result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_])) + elif isinstance(arg, (JoinFunction, UnionFunction)): + result.extend(self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.tokens)) + result.extend( + self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.functions.functions) + ) + result.extend(self.get_field_tokens_from_func_args(args=arg.condition)) elif isinstance(arg, Function): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) elif isinstance(arg, SortArg) and isinstance(arg.field, Field): diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena.py b/uncoder-core/app/translator/platforms/athena/renders/athena.py index a62e5b00..8550c94a 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena.py @@ -21,10 +21,10 @@ from app.translator.managers import render_manager from app.translator.platforms.athena.const import athena_details from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender -class AthenaFieldValue(SqlFieldValue): +class AthenaFieldValueRender(SqlFieldValueRender): details: PlatformDetails = athena_details @@ -35,7 +35,7 @@ class AthenaQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = AthenaFieldValue(or_token=or_token) + field_value_render = AthenaFieldValueRender(or_token=or_token) comment_symbol = "--" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py index 05826d08..6c0c1665 100644 --- a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager -class AQLFieldValue(BaseQueryFieldValue): +class AQLFieldValueRender(BaseFieldValueRender): str_value_manager = aql_str_value_manager @staticmethod @@ -127,8 +127,6 @@ class AQLQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = AQLFieldValue(or_token=or_token) - def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) extra_condition = log_source_signature.extra_condition diff --git a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py index b5994499..f8511d82 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.lucene.mapping import LuceneLogSourceSignature from app.translator.platforms.base.lucene.str_value_manager import lucene_str_value_manager -class LuceneFieldValue(BaseQueryFieldValue): +class LuceneFieldValueRender(BaseFieldValueRender): str_value_manager = lucene_str_value_manager @staticmethod diff --git a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py index b2c12068..74adf32b 100644 --- a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py @@ -21,11 +21,11 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.platforms.base.spl.escape_manager import spl_escape_manager -class SplFieldValue(BaseQueryFieldValue): +class SplFieldValueRender(BaseFieldValueRender): escape_manager = spl_escape_manager def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: diff --git a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py index 43904a1e..d69f1590 100644 --- a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py @@ -22,10 +22,10 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import LogSourceSignature -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender -class SqlFieldValue(BaseQueryFieldValue): +class SqlFieldValueRender(BaseFieldValueRender): def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py index 4101b825..8bcbe56f 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py @@ -23,14 +23,14 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import chronicle_query_details from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings -class ChronicleFieldValue(BaseQueryFieldValue): +class ChronicleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = chronicle_query_details escape_manager = chronicle_escape_manager @@ -109,6 +109,6 @@ class ChronicleQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = ChronicleFieldValue(or_token=or_token) + field_value_render = ChronicleFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py index aaa64384..1961c72b 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py @@ -26,12 +26,12 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details -from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValue, ChronicleQueryRender +from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValueRender, ChronicleQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated Chronicle Security rule." -class ChronicleRuleFieldValue(ChronicleFieldValue): +class ChronicleRuleFieldValueRender(ChronicleFieldValueRender): details: PlatformDetails = chronicle_rule_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -85,7 +85,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: class ChronicleSecurityRuleRender(ChronicleQueryRender): details: PlatformDetails = chronicle_rule_details or_token = "or" - field_value_map = ChronicleRuleFieldValue(or_token=or_token) + field_value_render = ChronicleRuleFieldValueRender(or_token=or_token) @staticmethod def prepare_title(title: str) -> str: diff --git a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py index 8c6630e9..3e5900cc 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings -class CrowdStrikeFieldValue(SplFieldValue): +class CrowdStrikeFieldValueRender(SplFieldValueRender): details = crowdstrike_query_details @@ -36,7 +36,7 @@ class CrowdStrikeQueryRender(SplQueryRender): platform_functions: CrowdStrikeFunctions = None or_token = "OR" - field_value_map = CrowdStrikeFieldValue(or_token=or_token) + field_value_render = CrowdStrikeFieldValueRender(or_token=or_token) comment_symbol = "`" def init_platform_functions(self) -> None: diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py index 09fad79b..0b7b20c4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -50,7 +50,7 @@ class ElasticSearchRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticSearchRuleFieldValue(or_token=or_token) + field_value_render = ElasticSearchRuleFieldValue(or_token=or_token) def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): @@ -109,7 +109,4 @@ def finalize_query( } ) rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py index 104b8ecc..9d7914ab 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py @@ -49,7 +49,7 @@ class ElastAlertRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticAlertRuleFieldValue(or_token=or_token) + field_value_render = ElasticAlertRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -75,7 +75,4 @@ def finalize_query( ) rule = rule.replace("", meta_info.title or _AUTOGENERATED_TEMPLATE) rule = rule.replace("", _SEVERITIES_MAP[meta_info.severity]) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py index 8d2db1d0..2e6a12f0 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -class ElasticSearchFieldValue(LuceneFieldValue): +class ElasticSearchFieldValue(LuceneFieldValueRender): details: PlatformDetails = elasticsearch_lucene_query_details @@ -34,4 +34,4 @@ class ElasticSearchQueryRender(LuceneQueryRender): mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = ElasticSearchFieldValue(or_token=or_token) + field_value_render = ElasticSearchFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py index c3b6a46a..53a4acf5 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py @@ -45,7 +45,7 @@ class KibanaRuleRender(ElasticSearchQueryRender): details: PlatformDetails = kibana_rule_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = KibanaFieldValue(or_token=or_token) + field_value_render = KibanaFieldValue(or_token=or_token) def finalize_query( self, @@ -74,7 +74,4 @@ def finalize_query( references=meta_info.references, ) rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py index 9a013dcf..d8421977 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py @@ -45,7 +45,7 @@ class XPackWatcherRuleRender(ElasticSearchQueryRender): details: PlatformDetails = xpack_watcher_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = XpackWatcherRuleFieldValue(or_token=or_token) + field_value_render = XpackWatcherRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -78,7 +78,4 @@ def finalize_query( rule["input"]["search"]["request"]["indices"] = indices rule["actions"]["send_email"]["email"]["subject"] = meta_info.title or _AUTOGENERATED_TEMPLATE rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index 65ca0b07..dfbc2ee6 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -18,6 +18,7 @@ from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.const import TOKEN_TYPE from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType @@ -28,9 +29,8 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue -from app.translator.core.tokenizer import TOKEN_TYPE from app.translator.managers import render_manager from app.translator.platforms.forti_siem.const import ( FORTI_SIEM_RULE, @@ -76,7 +76,7 @@ ] -class FortiSiemFieldValue(BaseQueryFieldValue): +class FortiSiemFieldValueRender(BaseFieldValueRender): details: PlatformDetails = forti_siem_rule_details str_value_manager = forti_siem_str_value_manager @@ -194,7 +194,7 @@ class FortiSiemRuleRender(PlatformQueryRender): group_token = "(%s)" - field_value_map = FortiSiemFieldValue(or_token=or_token) + field_value_render = FortiSiemFieldValueRender(or_token=or_token) @staticmethod def __is_negated_token(prev_token: TOKEN_TYPE) -> bool: @@ -244,7 +244,7 @@ def __replace_not_tokens(self, tokens: list[TOKEN_TYPE]) -> list[TOKEN_TYPE]: return tokens - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -324,11 +324,7 @@ def finalize_query( rule = rule.replace("", query) rule = rule.replace("", ", ".join(args_list)) rule = rule.replace("", self.get_attr_str(fields.copy())) - - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) @staticmethod def get_attr_str(fields: set[str]) -> str: diff --git a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py index 2bdf001e..986ddd93 100644 --- a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py +++ b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.graylog.const import graylog_details from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings -class GraylogFieldValue(LuceneFieldValue): +class GraylogFieldValue(LuceneFieldValueRender): details: PlatformDetails = graylog_details @@ -34,4 +34,4 @@ class GraylogQueryRender(LuceneQueryRender): mappings: GraylogMappings = graylog_mappings or_token = "OR" - field_value_map = GraylogFieldValue(or_token=or_token) + field_value_render = GraylogFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py index 0348bfb0..3c73c234 100644 --- a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py +++ b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender from app.translator.platforms.hunters.const import hunters_details from app.translator.platforms.hunters.mapping import HuntersMappings, hunters_mappings -class HuntersFieldValue(SqlFieldValue): +class HuntersFieldValueRender(SqlFieldValueRender): details: PlatformDetails = hunters_details @@ -35,7 +35,7 @@ class HuntersQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = HuntersFieldValue(or_token=or_token) + field_value_render = HuntersFieldValueRender(or_token=or_token) @staticmethod def _finalize_search_query(query: str) -> str: diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 624fa3d7..0cef42f2 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -30,7 +30,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_axon_query_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_query_escape_manager @@ -41,7 +41,7 @@ class LogRhythmRegexRenderException(BaseRenderException): ... -class LogRhythmAxonFieldValue(BaseQueryFieldValue): +class LogRhythmAxonFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logrhythm_axon_query_details escape_manager = logrhythm_query_escape_manager @@ -204,7 +204,7 @@ class LogRhythmAxonQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = LogRhythmAxonFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonFieldValueRender(or_token=or_token) mappings: LogRhythmAxonMappings = logrhythm_axon_mappings comment_symbol = "//" @@ -224,7 +224,7 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp mapped_fields = self.map_field(token.field, source_mapping) except StrictPlatformException: try: - return self.field_value_map.apply_field_value( + return self.field_value_render.apply_field_value( field=UNMAPPED_FIELD_DEFAULT_NAME, operator=token.operator, value=token.value ) except LogRhythmRegexRenderException as exc: @@ -232,20 +232,16 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp f"Uncoder does not support complex regexp for unmapped field:" f" {token.field.source_name} for LogRhythm Axon" ) from exc - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value(field=field, operator=token.operator, value=token.value) - for field in mapped_fields - ] - ) - return self.field_value_map.apply_field_value( - field=mapped_fields[0], operator=token.operator, value=token.value + return self.group_token % self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] ) return super().apply_token(token, source_mapping) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py index 20514140..2e68c2d1 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py @@ -29,7 +29,7 @@ from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import ( - LogRhythmAxonFieldValue, + LogRhythmAxonFieldValueRender, LogRhythmAxonQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -44,7 +44,7 @@ } -class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): +class LogRhythmAxonRuleFieldValueRender(LogRhythmAxonFieldValueRender): details: PlatformDetails = logrhythm_axon_rule_details escape_manager = logrhythm_rule_escape_manager @@ -53,7 +53,7 @@ class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): class LogRhythmAxonRuleRender(LogRhythmAxonQueryRender): details: PlatformDetails = logrhythm_axon_rule_details or_token = "or" - field_value_map = LogRhythmAxonRuleFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonRuleFieldValueRender(or_token=or_token) def finalize_query( self, @@ -93,7 +93,4 @@ def finalize_query( ] json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py index 9cb7cf05..e1ed4818 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py @@ -23,7 +23,7 @@ from app.translator.core.mapping import SourceMapping from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.escape_manager import logscale_escape_manager @@ -31,7 +31,7 @@ from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings -class LogScaleFieldValue(BaseQueryFieldValue): +class LogScaleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logscale_query_details escape_manager = logscale_escape_manager @@ -102,7 +102,7 @@ class LogScaleQueryRender(PlatformQueryRender): and_token = "" not_token = "not" - field_value_map = LogScaleFieldValue(or_token=or_token) + field_value_render = LogScaleFieldValueRender(or_token=or_token) def init_platform_functions(self) -> None: self.platform_functions = log_scale_functions @@ -123,8 +123,5 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = super().finalize_query(prefix=prefix, query=query, functions=functions) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py index 4b3af0fb..a6628045 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py @@ -26,13 +26,13 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details -from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValue, LogScaleQueryRender +from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValueRender, LogScaleQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Falcon LogScale Alert" -class LogScaleAlertFieldValue(LogScaleFieldValue): +class LogScaleAlertFieldValueRender(LogScaleFieldValueRender): details: PlatformDetails = logscale_alert_details @@ -40,7 +40,7 @@ class LogScaleAlertFieldValue(LogScaleFieldValue): class LogScaleAlertRender(LogScaleQueryRender): details: PlatformDetails = logscale_alert_details or_token = "or" - field_value_map = LogScaleAlertFieldValue(or_token=or_token) + field_value_render = LogScaleAlertFieldValueRender(or_token=or_token) def finalize_query( self, @@ -70,8 +70,5 @@ def finalize_query( mitre_attack=mitre_attack, ) - json_query = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_query + rendered_not_supported - return json_query + rule_str = json.dumps(rule, indent=4, sort_keys=False) + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py index 7b7a3779..38617b55 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py @@ -23,12 +23,12 @@ from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) -class MicrosoftDefenderFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftDefenderFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_defender_details @@ -38,7 +38,7 @@ class MicrosoftDefenderQueryRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_defender_details platform_functions: MicrosoftFunctions = None or_token = "or" - field_value_map = MicrosoftDefenderFieldValue(or_token=or_token) + field_value_render = MicrosoftDefenderFieldValueRender(or_token=or_token) is_strict_mapping = True diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py index 3153f8d4..7ef6f1f9 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py @@ -22,7 +22,7 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.escape_manager import microsoft_escape_manager @@ -30,7 +30,7 @@ from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings -class MicrosoftSentinelFieldValue(BaseQueryFieldValue): +class MicrosoftSentinelFieldValueRender(BaseFieldValueRender): details: PlatformDetails = microsoft_sentinel_query_details escape_manager = microsoft_escape_manager @@ -128,7 +128,7 @@ class MicrosoftSentinelQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = MicrosoftSentinelFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelFieldValueRender(or_token=or_token) mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings comment_symbol = "//" diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index e2fdb81f..b5631ef5 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -28,7 +28,7 @@ from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -42,7 +42,7 @@ } -class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftSentinelRuleFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_sentinel_rule_details @@ -50,7 +50,7 @@ class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): class MicrosoftSentinelRuleRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_sentinel_rule_details or_token = "or" - field_value_map = MicrosoftSentinelRuleFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelRuleFieldValueRender(or_token=or_token) def __create_mitre_threat(self, meta_info: MetaInfoContainer) -> tuple[list, list]: tactics = set() @@ -92,7 +92,4 @@ def finalize_query( rule["tactics"] = mitre_tactics rule["techniques"] = mitre_techniques json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py index 1d2145a7..3298c106 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py @@ -24,12 +24,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.opensearch.const import opensearch_query_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -class OpenSearchFieldValue(LuceneFieldValue): +class OpenSearchFieldValueRender(LuceneFieldValueRender): details: PlatformDetails = opensearch_query_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -102,4 +102,4 @@ class OpenSearchQueryRender(LuceneQueryRender): mappings: OpenSearchMappings = opensearch_mappings or_token = "OR" - field_value_map = OpenSearchFieldValue(or_token=or_token) + field_value_render = OpenSearchFieldValueRender(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py index 3f68e6c6..106a1fe1 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -30,13 +30,13 @@ from app.translator.managers import render_manager from app.translator.platforms.opensearch.const import OPENSEARCH_RULE, opensearch_rule_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValue, OpenSearchQueryRender +from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValueRender, OpenSearchQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated AWS OpenSearch Rule" _SEVERITIES_MAP = {SeverityType.critical: "5", SeverityType.high: "4", SeverityType.medium: "3", SeverityType.low: "2"} -class OpenSearchRuleFieldValue(OpenSearchFieldValue): +class OpenSearchRuleFieldValueRender(OpenSearchFieldValueRender): details: PlatformDetails = opensearch_rule_details @@ -49,7 +49,7 @@ class OpenSearchRuleRender(OpenSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = OpenSearchRuleFieldValue(or_token=or_token) + field_value_render = OpenSearchRuleFieldValueRender(or_token=or_token) def __init__(self): super().__init__() @@ -76,10 +76,7 @@ def finalize_query( rule["triggers"][0]["severity"] = _SEVERITIES_MAP[meta_info.severity] rule["triggers"][0]["actions"][0]["message_template"]["source"] = str(source).replace(", ", ",\n") rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index 54f50916..d3300abd 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -20,9 +20,11 @@ from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import preset_log_source_str_ctx_var +from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager from app.translator.platforms.palo_alto.const import cortex_xql_query_details @@ -35,7 +37,7 @@ from app.translator.platforms.palo_alto.str_value_manager import cortex_xql_str_value_manager -class CortexXQLFieldValue(BaseQueryFieldValue): +class CortexXQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = cortex_xql_query_details str_value_manager = cortex_xql_str_value_manager @@ -132,6 +134,17 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"_raw_log contains {self._pre_process_value(field ,value, value_type=ValueType.value, wrap_str=True)}" +class CortexXQLFieldFieldRender(BaseFieldFieldRender): + operators_map: ClassVar[dict[str, str]] = { + OperatorType.EQ: "=", + OperatorType.NOT_EQ: "!=", + OperatorType.LT: "<", + OperatorType.LTE: "<=", + OperatorType.GT: ">", + OperatorType.GTE: ">=", + } + + @render_manager.register class CortexXQLQueryRender(PlatformQueryRender): details: PlatformDetails = cortex_xql_query_details @@ -149,7 +162,8 @@ class CortexXQLQueryRender(PlatformQueryRender): not_token = "not" query_parts_delimiter = "\n" - field_value_map = CortexXQLFieldValue(or_token=or_token) + field_field_render = CortexXQLFieldFieldRender() + field_value_render = CortexXQLFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = False @@ -171,7 +185,8 @@ def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, functions_prefix: str = "") -> str: functions_prefix = f"{functions_prefix} | " if functions_prefix else "" - return f"{functions_prefix}{log_source_signature}" + log_source_str = preset_log_source_str_ctx_var.get() or str(log_source_signature) + return f"{functions_prefix}{log_source_str}" @staticmethod def _finalize_search_query(query: str) -> str: diff --git a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py index 0f06fb40..cf4a7d51 100644 --- a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py @@ -19,14 +19,15 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.aql.renders.aql import AQLFieldValue, AQLQueryRender +from app.translator.platforms.base.aql.renders.aql import AQLFieldValueRender, AQLQueryRender from app.translator.platforms.qradar.const import qradar_query_details -class QradarFieldValue(AQLFieldValue): +class QradarFieldValueRender(AQLFieldValueRender): details: PlatformDetails = qradar_query_details @render_manager.register class QradarQueryRender(AQLQueryRender): details: PlatformDetails = qradar_query_details + field_value_render = QradarFieldValueRender(or_token=AQLQueryRender.or_token) diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index dc33a507..856fd4a3 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -281,10 +281,10 @@ def __get_source_mapping(self, source_mapping_ids: list[str]) -> SourceMapping: return self.mappings.get_source_mapping(DEFAULT_MAPPING_NAME) - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: raise NotImplementedError - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: self.reset_counters() meta_info = query_container.meta_info @@ -316,6 +316,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py index f9404cac..e14c6bfc 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings -class SplunkFieldValue(SplFieldValue): +class SplunkFieldValueRender(SplFieldValueRender): details: PlatformDetails = splunk_query_details @@ -35,7 +35,7 @@ class SplunkQueryRender(SplQueryRender): or_token = "OR" - field_value_map = SplunkFieldValue(or_token=or_token) + field_value_render = SplunkFieldValueRender(or_token=or_token) mappings: SplunkMappings = splunk_mappings platform_functions: SplunkFunctions = None diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py index ef0d097d..5dc2096a 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py @@ -25,14 +25,14 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details -from app.translator.platforms.splunk.renders.splunk import SplunkFieldValue, SplunkQueryRender +from app.translator.platforms.splunk.renders.splunk import SplunkFieldValueRender, SplunkQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Splunk Alert" _SEVERITIES_MAP = {SeverityType.critical: "4", SeverityType.high: "3", SeverityType.medium: "2", SeverityType.low: "1"} -class SplunkAlertFieldValue(SplunkFieldValue): +class SplunkAlertFieldValueRender(SplunkFieldValueRender): details: PlatformDetails = splunk_alert_details @@ -40,7 +40,7 @@ class SplunkAlertFieldValue(SplunkFieldValue): class SplunkAlertRender(SplunkQueryRender): details: PlatformDetails = splunk_alert_details or_token = "OR" - field_value_map = SplunkAlertFieldValue(or_token=or_token) + field_value_render = SplunkAlertFieldValueRender(or_token=or_token) @staticmethod def __create_mitre_threat(meta_info: MetaInfoContainer) -> dict: @@ -74,7 +74,4 @@ def finalize_query( if mitre_techniques: mitre_str = f"action.correlationsearch.annotations = {mitre_techniques})" rule = rule.replace("", mitre_str) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) From fb8f50e4baa0ef41152b7c5cd8ecdac59094c041 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Mon, 24 Jun 2024 13:11:31 +0300 Subject: [PATCH 2/6] fix --- uncoder-core/app/translator/core/render.py | 2 +- .../platforms/logrhythm_axon/renders/logrhythm_axon_query.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 636f718f..caaaa74d 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -287,7 +287,7 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) return "".join(result_values) - def wrap_with_meta_info(self, query: str, meta_info: MetaInfoContainer) -> str: + def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str: if wrap_query_with_meta_info_ctx_var.get() and meta_info and (meta_info.id or meta_info.title): meta_info_dict = { "name: ": meta_info.title, diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 0cef42f2..4a288491 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -232,12 +232,13 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp f"Uncoder does not support complex regexp for unmapped field:" f" {token.field.source_name} for LogRhythm Axon" ) from exc - return self.group_token % self.logical_operators_map[LogicalOperatorType.OR].join( + joined = self.logical_operators_map[LogicalOperatorType.OR].join( [ self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) for field in mapped_fields ] ) + return self.group_token % joined if len(mapped_fields) > 1 else joined return super().apply_token(token, source_mapping) From 1719f09f1bd4848312afa0eded08d9d46e4e5287 Mon Sep 17 00:00:00 2001 From: "oleksandr.volha" Date: Mon, 24 Jun 2024 13:16:37 +0300 Subject: [PATCH 3/6] add models --- .../translator/core/models/functions/join.py | 26 +++++++++++++++++++ .../translator/core/models/functions/union.py | 12 +++++++++ 2 files changed, 38 insertions(+) create mode 100644 uncoder-core/app/translator/core/models/functions/join.py create mode 100644 uncoder-core/app/translator/core/models/functions/union.py diff --git a/uncoder-core/app/translator/core/models/functions/join.py b/uncoder-core/app/translator/core/models/functions/join.py new file mode 100644 index 00000000..0f44da68 --- /dev/null +++ b/uncoder-core/app/translator/core/models/functions/join.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass, field +from typing import Union + +from app.translator.core.custom_types.functions import FunctionType +from app.translator.core.models.field import Alias, Field +from app.translator.core.models.functions.base import Function +from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.tools.custom_enum import CustomEnum + + +class JoinType(CustomEnum): + inner = "inner" + left = "left" + right = "right" + cross = "cross" + + +@dataclass +class JoinFunction(Function): + name: str = FunctionType.join + alias: Alias = None + type_: str = JoinType.inner + tokenized_query_container: TokenizedQueryContainer = None + condition: list[Union[Alias, Field, Identifier]] = field(default_factory=list) + preset_log_source_str: str = None diff --git a/uncoder-core/app/translator/core/models/functions/union.py b/uncoder-core/app/translator/core/models/functions/union.py new file mode 100644 index 00000000..ef4d4179 --- /dev/null +++ b/uncoder-core/app/translator/core/models/functions/union.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass + +from app.translator.core.custom_types.functions import FunctionType +from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_container import TokenizedQueryContainer + + +@dataclass +class UnionFunction(Function): + name: str = FunctionType.union + tokenized_query_container: TokenizedQueryContainer = None + preset_log_source_str: str = None From 93ceff9996799e093a1883534657b1232c134fd9 Mon Sep 17 00:00:00 2001 From: "oleksandr.volha" Date: Mon, 24 Jun 2024 13:19:21 +0300 Subject: [PATCH 4/6] add name --- uncoder-core/app/translator/core/custom_types/functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/uncoder-core/app/translator/core/custom_types/functions.py b/uncoder-core/app/translator/core/custom_types/functions.py index bb31e77f..8e609a03 100644 --- a/uncoder-core/app/translator/core/custom_types/functions.py +++ b/uncoder-core/app/translator/core/custom_types/functions.py @@ -28,6 +28,7 @@ class FunctionType(CustomEnum): bin = "bin" eval = "eval" fields = "fields" + join = "join" rename = "rename" search = "search" sort_limit = "sort_limit" From eccbae370774bcfd3976e40a7421b818b9d92148 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Wed, 26 Jun 2024 14:30:02 +0300 Subject: [PATCH 5/6] resolve conflicts --- .../app/translator/core/custom_types/time.py | 14 ++++++++++ .../app/translator/core/exceptions/core.py | 18 +++++++++++-- .../app/translator/core/models/field.py | 5 ++++ uncoder-core/app/translator/core/render.py | 8 +++++- .../platforms/palo_alto_cortex/default.yml | 1 + .../platforms/palo_alto_cortex/dns.yml | 3 ++- .../windows_registry_event.yml | 3 ++- .../mappings/platforms/qradar/default.yml | 17 +++++++++--- .../mappings/platforms/qradar/dns.yml | 3 ++- .../mappings/platforms/qradar/proxy.yml | 1 + .../platforms/qradar/windows_security.yml | 5 +++- .../platforms/splunk/aws_cloudtrail.yml | 4 +-- .../mappings/platforms/splunk/aws_eks.yml | 4 +-- .../splunk/azure_AzureDiagnostics.yml | 4 +-- .../splunk/azure_BehaviorAnalytics.yml | 4 +-- .../azure_aadnoninteractiveusersigninlogs.yml | 4 +-- .../platforms/splunk/azure_azureactivity.yml | 4 +-- .../platforms/splunk/azure_azuread.yml | 4 +-- .../platforms/splunk/azure_signinlogs.yml | 4 +-- .../mappings/platforms/splunk/firewall.yml | 4 +-- .../platforms/splunk/gcp_gcp.audit.yml | 2 +- .../mappings/platforms/splunk/gcp_pubsub.yml | 2 +- .../platforms/splunk/linux_auditd.yml | 4 +-- .../mappings/platforms/splunk/okta_okta.yml | 4 +-- .../platforms/splunk/windows_bits_client.yml | 4 +-- .../platforms/splunk/windows_dns_query.yml | 4 +-- .../platforms/splunk/windows_driver_load.yml | 4 +-- .../platforms/splunk/windows_file_access.yml | 4 +-- .../platforms/splunk/windows_file_change.yml | 4 +-- .../platforms/splunk/windows_file_create.yml | 4 +-- .../platforms/splunk/windows_file_delete.yml | 4 +-- .../platforms/splunk/windows_file_event.yml | 4 +-- .../platforms/splunk/windows_file_rename.yml | 4 +-- .../platforms/splunk/windows_image_load.yml | 4 +-- .../platforms/splunk/windows_ldap_debug.yml | 4 +-- .../splunk/windows_network_connection.yml | 4 +-- .../platforms/splunk/windows_ntlm.yml | 4 +-- .../splunk/windows_registry_event.yml | 4 +-- .../platforms/splunk/windows_sysmon.yml | 4 +-- .../platforms/splunk/windows_wmi_event.yml | 4 +-- .../palo_alto/renders/cortex_xsiam.py | 26 +++++++++++++++++++ .../app/translator/platforms/sigma/mapping.py | 6 ++--- .../translator/platforms/splunk/mapping.py | 4 +-- 43 files changed, 155 insertions(+), 71 deletions(-) diff --git a/uncoder-core/app/translator/core/custom_types/time.py b/uncoder-core/app/translator/core/custom_types/time.py index 1d5f15b8..4cdc71fe 100644 --- a/uncoder-core/app/translator/core/custom_types/time.py +++ b/uncoder-core/app/translator/core/custom_types/time.py @@ -7,3 +7,17 @@ class TimeFrameType(CustomEnum): days = "days" hours = "hours" minutes = "minutes" + + +class TimePartType(CustomEnum): + day = "day" + day_of_week = "day_of_week" + day_of_year = "day_of_year" + hour = "hour" + microsecond = "microsecond" + millisecond = "millisecond" + minute = "minute" + month = "month" + quarter = "quarter" + second = "second" + year = "year" diff --git a/uncoder-core/app/translator/core/exceptions/core.py b/uncoder-core/app/translator/core/exceptions/core.py index 68c66962..47810576 100644 --- a/uncoder-core/app/translator/core/exceptions/core.py +++ b/uncoder-core/app/translator/core/exceptions/core.py @@ -1,3 +1,6 @@ +from typing import Optional + + class NotImplementedException(BaseException): ... @@ -7,8 +10,19 @@ class BasePlatformException(BaseException): class StrictPlatformException(BasePlatformException): - def __init__(self, platform_name: str, field_name: str): - message = f"Platform {platform_name} has strict mapping. Source field {field_name} has no mapping." + field_name: str = None + + def __init__( + self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None + ): + message = ( + f"Platform {platform_name} has strict mapping. " + f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping." + f" Mapping file: {mapping}." + if mapping + else "" + ) + self.field_name = field_name super().__init__(message) diff --git a/uncoder-core/app/translator/core/models/field.py b/uncoder-core/app/translator/core/models/field.py index 0c430382..95556dcc 100644 --- a/uncoder-core/app/translator/core/models/field.py +++ b/uncoder-core/app/translator/core/models/field.py @@ -76,6 +76,11 @@ def value(self) -> Union[int, str, StrValue, list[Union[int, str, StrValue]]]: return self.values[0] return self.values + @value.setter + def value(self, new_value: Union[int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: + self.values = [] + self.__add_value(new_value) + def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -> None: if value and isinstance(value, (list, tuple)): for v in value: diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index caaaa74d..b66f4430 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -283,8 +283,14 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str: result_values = [] + unmapped_fields = set() for token in tokens: - result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) + try: + result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) + except StrictPlatformException as err: + unmapped_fields.add(err.field_name) + if unmapped_fields: + raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields)) return "".join(result_values) def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str: diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml index f6b25023..fa904aaf 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml @@ -125,3 +125,4 @@ field_mapping: SourceOS: xdm.source.host.os DestinationOS: xdm.target.host.os url_category: xdm.network.http.url_category + EventSeverity: xdm.alert.severity diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml index e489fd50..e279a60a 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml @@ -10,4 +10,5 @@ field_mapping: #dns-record: dns-record dns_query_name: xdm.network.dns.dns_question.name QueryName: xdm.network.dns.dns_question.name - query: xdm.network.dns.dns_question.name \ No newline at end of file + query: xdm.network.dns.dns_question.name + dns-record-type: xdm.network.dns.dns_question.type \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml index 86110049..04abb36b 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml @@ -28,4 +28,5 @@ field_mapping: ParentIntegrityLevel: causality_actor_process_integrity_level ParentLogonId: causality_actor_process_logon_id ParentProduct: causality_actor_process_signature_product - ParentCompany: causality_actor_process_signature_vendor \ No newline at end of file + ParentCompany: causality_actor_process_signature_vendor + EventType: event_sub_type \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml index 6e798034..004e10c7 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml @@ -13,9 +13,12 @@ field_mapping: dst-port: - DstPort - DestinationPort + - remoteport dst-hostname: DstHost src-hostname: SrcHost - src-port: SourcePort + src-port: + - SourcePort + - localport src-ip: - sourceip - source_ip @@ -27,11 +30,14 @@ field_mapping: - destination_ip - destinationIP - destinationaddress + - destination User: - userName - EventUserName CommandLine: Command - Protocol: IPProtocol + Protocol: + - IPProtocol + - protocol Application: - Application - application @@ -57,6 +63,7 @@ field_mapping: SourceMAC: - SourceMAC - MAC + - sourceMAC DestinationMAC: DestinationMAC SourceOS: - SourceOS @@ -64,4 +71,8 @@ field_mapping: DestinationOS: DestinationOS TargetUserName: DestinationUserName SourceUserName: SourceUserName - url_category: XForceCategoryByURL \ No newline at end of file + url_category: XForceCategoryByURL + EventSeverity: EventSeverity + Source: + - Source + - source \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml b/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml index 048a4bd3..d9aad78e 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml @@ -12,4 +12,5 @@ field_mapping: dns-query: URL parent-domain: parent-domain dns-answer: dns-answer - dns-record: URL \ No newline at end of file + dns-record: URL + dns-record-type: DNSRecordType \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml index 58393ac0..193bc79c 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml @@ -24,6 +24,7 @@ field_mapping: cs-host: - UrlHost - URL Host + - URL Domain cs-referrer: - URL Referrer - Referrer URL diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml index 7d01b97e..53b37952 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml @@ -41,7 +41,9 @@ field_mapping: LinkName: LinkName MemberName: MemberName MemberSid: MemberSid - NewProcessName: Process Name + NewProcessName: + - Process Name + - New Process Name ObjectClass: ObjectClass ObjectName: - Object Name @@ -122,6 +124,7 @@ field_mapping: ServiceFileName: - Service Filename - ServiceFileName + - Service File Name SecurityDescriptor: SecurityDescriptor ServiceName: Service Name ShareName: diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml b/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml index acd62dbc..96bb06b8 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml @@ -3,10 +3,10 @@ source: aws_cloudtrail log_source: - source_type: [aws:cloudtrail] + sourcetype: [aws:cloudtrail] default_log_source: - source_type: aws:cloudtrail + sourcetype: aws:cloudtrail field_mapping: eventSource: eventSource diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml b/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml index 32302e30..38e225d7 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml @@ -3,10 +3,10 @@ source: aws_eks log_source: - source_type: [aws:*] + sourcetype: [aws:*] default_log_source: - source_type: aws:* + sourcetype: aws:* field_mapping: annotations.authorization.k8s.io\/decision: annotations.authorization.k8s.io\/decision diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml index 5cff60da..90fd75a1 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml @@ -3,10 +3,10 @@ source: azure_AzureDiagnostics log_source: - source_type: [azure:*] + sourcetype: [azure:*] default_log_source: - source_type: azure:* + sourcetype: azure:* field_mapping: ResultDescription: ResultDescription diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml index 379004da..e1f17620 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml @@ -3,10 +3,10 @@ source: azure_BehaviorAnalytics log_source: - source_type: [azure:*] + sourcetype: [azure:*] default_log_source: - source_type: azure:* + sourcetype: azure:* field_mapping: ActionType: ActionType diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml index 3e994fc5..ad6bb5eb 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml @@ -3,10 +3,10 @@ source: azure_aadnoninteractiveusersigninlogs log_source: - source_type: [azure:*] + sourcetype: [azure:*] default_log_source: - source_type: azure:* + sourcetype: azure:* field_mapping: UserAgent: UserAgent diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml index d3623983..337125f4 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml @@ -3,10 +3,10 @@ source: azure_azureactivity log_source: - source_type: [mscs:azure:*, azure:*] + sourcetype: [mscs:azure:*, azure:*] default_log_source: - source_type: mscs:azure:* + sourcetype: mscs:azure:* field_mapping: ActivityStatus: ActivityStatus diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml index 5f393c91..69e3d195 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml @@ -3,10 +3,10 @@ source: azure_azuread log_source: - source_type: [azure:aad:*] + sourcetype: [azure:aad:*] default_log_source: - source_type: azure:aad:* + sourcetype: azure:aad:* field_mapping: ActivityDisplayName: ActivityDisplayName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml index 23b7569b..4f669d89 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml @@ -3,10 +3,10 @@ source: azure_signinlogs log_source: - source_type: [azure:aad:*] + sourcetype: [azure:aad:*] default_log_source: - source_type: azure:aad:* + sourcetype: azure:aad:* field_mapping: AppDisplayName: AppDisplayName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml b/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml index f40ef682..ed886d9c 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml @@ -3,11 +3,11 @@ source: firewall log_source: - source_type: [fortigate_traffic] + sourcetype: [fortigate_traffic] index: [fortigate] default_log_source: - source_type: fortigate_traffic + sourcetype: fortigate_traffic index: fortigate field_mapping: diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml index ef92fb58..be54b882 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml @@ -3,7 +3,7 @@ source: gcp_gcp.audit log_source: - source_type: [google:gcp:*] + sourcetype: [google:gcp:*] default_log_source: index: google:gcp:* diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml index 7ab8483c..dbfd2736 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml @@ -3,7 +3,7 @@ source: gcp_pubsub log_source: - source_type: [google:gcp:*] + sourcetype: [google:gcp:*] default_log_source: index: google:gcp:* diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml b/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml index ee3ac161..afd115b0 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml @@ -3,10 +3,10 @@ source: linux_auditd log_source: - source_type: [linux:audit] + sourcetype: [linux:audit] default_log_source: - source_type: linux:audit + sourcetype: linux:audit field_mapping: a0: a0 diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml b/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml index 3ee6d0e1..3f55621f 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml @@ -3,10 +3,10 @@ source: okta_okta log_source: - source_type: [OktaIM2:*] + sourcetype: [OktaIM2:*] default_log_source: - source_type: OktaIM2:* + sourcetype: OktaIM2:* field_mapping: client.user.id: client.user.id diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml index 014287eb..babbd610 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml @@ -2,10 +2,10 @@ platform: Splunk source: windows_bits_client log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational field_mapping: LocalName: LocalName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml index 698e62cc..d8e40100 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml @@ -4,11 +4,11 @@ source: windows_dns_query log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: Image: Image diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml index f8248b8e..86b76510 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml @@ -4,11 +4,11 @@ source: windows_driver_load log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: ImageLoaded: ImageLoaded diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml index 5c1c64f2..48ab5786 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml @@ -4,11 +4,11 @@ source: windows_file_access log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml index 0114b7e0..f45393aa 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml @@ -4,11 +4,11 @@ source: windows_file_change log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml index d9b0d8c0..485ea463 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml @@ -4,11 +4,11 @@ source: windows_file_create log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml index 8b82cc38..13660235 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml @@ -4,11 +4,11 @@ source: windows_file_delete log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml index 278b9b30..ed0855d3 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml @@ -4,11 +4,11 @@ source: windows_file_event log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml index 10390535..dae50085 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml @@ -4,11 +4,11 @@ source: windows_file_rename log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml index 8f427639..3cc22f55 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml @@ -4,11 +4,11 @@ source: windows_image_load log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: Image: Image diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml index 8fc85d34..f8241ba4 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml @@ -3,10 +3,10 @@ source: windows_ldap_debug log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug] + sourcetype: [XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug + sourcetype: XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug field_mapping: EventID: EventID diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml index d8260810..7a92b32c 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml @@ -4,11 +4,11 @@ source: windows_network_connection log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: Image: Image diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml index 3ea2c8ea..7902c0fe 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml @@ -3,10 +3,10 @@ source: windows_ntlm log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-NTLM/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-NTLM/Operational] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-NTLM/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-NTLM/Operational field_mapping: WorkstationName: WorkstationName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml index 8cbe38f3..a2169567 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml @@ -4,11 +4,11 @@ source: windows_registry_event log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: TargetObject: TargetObject diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml index a361471a..89bf98e0 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml @@ -3,11 +3,11 @@ source: windows_sysmon log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CommandLine: CommandLine diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml index 5e1e47bd..b1e415d0 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml @@ -3,10 +3,10 @@ source: windows_wmi_event log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational field_mapping: Destination: Destination diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index d3300abd..1f2f7d04 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -23,6 +23,9 @@ from app.translator.core.context_vars import preset_log_source_str_ctx_var from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType +from app.translator.core.mapping import SourceMapping +from app.translator.core.models.field import FieldValue, Keyword +from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue @@ -36,6 +39,16 @@ ) from app.translator.platforms.palo_alto.str_value_manager import cortex_xql_str_value_manager +SOURCE_MAPPING_TO_FIELD_VALUE_MAP = { + "windows_registry_event": { + "EventType": { + "SetValue": "REGISTRY_SET_VALUE", + "DeleteValue": "REGISTRY_DELETE_VALUE", + "CreateKey": "REGISTRY_CREATE_KEY", + } + } +} + class CortexXQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = cortex_xql_query_details @@ -188,6 +201,19 @@ def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, fun log_source_str = preset_log_source_str_ctx_var.get() or str(log_source_signature) return f"{functions_prefix}{log_source_str}" + def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + if isinstance(token, FieldValue): + field_name = token.field.source_name + if values_map := SOURCE_MAPPING_TO_FIELD_VALUE_MAP.get(source_mapping.source_id, {}).get(field_name): + values_to_update = [] + for token_value in token.values: + mapped_value: str = values_map.get(token_value, token_value) + values_to_update.append( + StrValue(value=mapped_value, split_value=mapped_value.split()) if mapped_value else token_value + ) + token.value = values_to_update + return super().apply_token(token=token, source_mapping=source_mapping) + @staticmethod def _finalize_search_query(query: str) -> str: return f"| filter {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/sigma/mapping.py b/uncoder-core/app/translator/platforms/sigma/mapping.py index 3f23700d..1af791ac 100644 --- a/uncoder-core/app/translator/platforms/sigma/mapping.py +++ b/uncoder-core/app/translator/platforms/sigma/mapping.py @@ -19,9 +19,9 @@ def __init__( def is_suitable( self, service: Optional[list[str]], product: Optional[list[str]], category: Optional[list[str]] ) -> bool: - product_match = set(product or []).issubset(self.products) if product else False - category_match = set(category or []).issubset(self.categories) if category else False - service_match = set(service or []).issubset(self.services) if service else False + product_match = set(product_.lower() for product_ in product or []).issubset(self.products) if product else False + category_match = set(category_.lower() for category_ in category or []).issubset(self.categories) if category else False + service_match = set(service_.lower() for service_ in service or [] or []).issubset(self.services) if service else False if not product and not service: return category_match return product_match and service_match or product_match and category_match diff --git a/uncoder-core/app/translator/platforms/splunk/mapping.py b/uncoder-core/app/translator/platforms/splunk/mapping.py index 2f9c4a8d..1851b8af 100644 --- a/uncoder-core/app/translator/platforms/splunk/mapping.py +++ b/uncoder-core/app/translator/platforms/splunk/mapping.py @@ -42,8 +42,8 @@ def prepare_log_source_signature(self, mapping: dict) -> SplunkLogSourceSignatur default_log_source = mapping["default_log_source"] return SplunkLogSourceSignature( sources=log_source.get("source"), - source_types=log_source.get("source_type"), - source_categories=log_source.get("source_category"), + source_types=log_source.get("sourcetype"), + source_categories=log_source.get("sourcecategory"), indices=log_source.get("index"), default_source=default_log_source, ) From 8d954b3b7599af76a35bebc3553fdd79583a5b2e Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Thu, 27 Jun 2024 10:27:40 +0300 Subject: [PATCH 6/6] fix --- uncoder-core/app/translator/core/tokenizer.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index 2ecbc39c..7530af5f 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -344,11 +344,7 @@ def get_field_tokens_from_func_args( # noqa: PLR0912 result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_])) elif isinstance(arg, (JoinFunction, UnionFunction)): - result.extend(self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.tokens)) - result.extend( - self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.functions.functions) - ) - result.extend(self.get_field_tokens_from_func_args(args=arg.condition)) + continue elif isinstance(arg, Function): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) elif isinstance(arg, SortArg) and isinstance(arg.field, Field): pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy