diff --git a/uncoder-core/app/translator/core/const.py b/uncoder-core/app/translator/core/const.py new file mode 100644 index 00000000..a8788ada --- /dev/null +++ b/uncoder-core/app/translator/core/const.py @@ -0,0 +1,6 @@ +from typing import Union + +from app.translator.core.models.field import Alias, Field, FieldValue, Keyword +from app.translator.core.models.identifier import Identifier + +TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias] diff --git a/uncoder-core/app/translator/core/context_vars.py b/uncoder-core/app/translator/core/context_vars.py index 591883d8..8ff6ccfb 100644 --- a/uncoder-core/app/translator/core/context_vars.py +++ b/uncoder-core/app/translator/core/context_vars.py @@ -1,4 +1,10 @@ from contextvars import ContextVar +from typing import Optional return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False) """Set to True to return only first query if rendered multiple options""" + +wrap_query_with_meta_info_ctx_var: ContextVar[bool] = ContextVar("wrap_query_with_meta_info_ctx_var", default=True) +"""Set to False not to wrap query with meta info commentary""" + +preset_log_source_str_ctx_var: ContextVar[Optional[str]] = ContextVar("preset_log_source_str_ctx_var", default=None) diff --git a/uncoder-core/app/translator/core/custom_types/functions.py b/uncoder-core/app/translator/core/custom_types/functions.py index bb31e77f..8e609a03 100644 --- a/uncoder-core/app/translator/core/custom_types/functions.py +++ b/uncoder-core/app/translator/core/custom_types/functions.py @@ -28,6 +28,7 @@ class FunctionType(CustomEnum): bin = "bin" eval = "eval" fields = "fields" + join = "join" rename = "rename" search = "search" sort_limit = "sort_limit" diff --git a/uncoder-core/app/translator/core/models/field.py b/uncoder-core/app/translator/core/models/field.py index 2576f93a..95556dcc 100644 --- a/uncoder-core/app/translator/core/models/field.py +++ b/uncoder-core/app/translator/core/models/field.py @@ -37,6 +37,22 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma self.__generic_names_map = generic_names_map +class FieldField: + def __init__( + self, + source_name_left: str, + operator: Identifier, + source_name_right: str, + is_alias_left: bool = False, + is_alias_right: bool = False, + ): + self.field_left = Field(source_name=source_name_left) + self.alias_left = Alias(name=source_name_left) if is_alias_left else None + self.operator = operator + self.field_right = Field(source_name=source_name_right) + self.alias_right = Alias(name=source_name_right) if is_alias_right else None + + class FieldValue: def __init__( self, diff --git a/uncoder-core/app/translator/core/models/functions/join.py b/uncoder-core/app/translator/core/models/functions/join.py new file mode 100644 index 00000000..0f44da68 --- /dev/null +++ b/uncoder-core/app/translator/core/models/functions/join.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass, field +from typing import Union + +from app.translator.core.custom_types.functions import FunctionType +from app.translator.core.models.field import Alias, Field +from app.translator.core.models.functions.base import Function +from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.tools.custom_enum import CustomEnum + + +class JoinType(CustomEnum): + inner = "inner" + left = "left" + right = "right" + cross = "cross" + + +@dataclass +class JoinFunction(Function): + name: str = FunctionType.join + alias: Alias = None + type_: str = JoinType.inner + tokenized_query_container: TokenizedQueryContainer = None + condition: list[Union[Alias, Field, Identifier]] = field(default_factory=list) + preset_log_source_str: str = None diff --git a/uncoder-core/app/translator/core/models/functions/union.py b/uncoder-core/app/translator/core/models/functions/union.py new file mode 100644 index 00000000..ef4d4179 --- /dev/null +++ b/uncoder-core/app/translator/core/models/functions/union.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass + +from app.translator.core.custom_types.functions import FunctionType +from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_container import TokenizedQueryContainer + + +@dataclass +class UnionFunction(Function): + name: str = FunctionType.union + tokenized_query_container: TokenizedQueryContainer = None + preset_log_source_str: str = None diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index dccfc180..0d90f237 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -3,11 +3,11 @@ from datetime import datetime from typing import Optional +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import DEFAULT_MAPPING_NAME from app.translator.core.models.field import Field from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.tokenizer import TOKEN_TYPE class MetaInfoContainer: diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 7cc10ec1..18b50739 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping @@ -28,7 +29,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer -from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer +from app.translator.core.tokenizer import QueryTokenizer class QueryParser(ABC): diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index b326cf96..b66f4430 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -16,13 +16,14 @@ limitations under the License. ----------------------------------------------------------------- """ - +import itertools from abc import ABC, abstractmethod from collections.abc import Callable from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.const import TOKEN_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -30,22 +31,21 @@ from app.translator.core.exceptions.parser import UnsupportedOperatorException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function, RenderedFunctions from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.str_value_manager import StrValue, StrValueManager -from app.translator.core.tokenizer import TOKEN_TYPE -class BaseQueryFieldValue(ABC): +class BaseFieldValueRender(ABC): details: PlatformDetails = None escape_manager: EscapeManager = None str_value_manager: StrValueManager = None def __init__(self, or_token: str): - self.field_value: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { + self.modifiers_map: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { OperatorType.EQ: self.equal_modifier, OperatorType.NOT_EQ: self.not_equal_modifier, OperatorType.LT: self.less_modifier, @@ -155,11 +155,20 @@ def apply_value(self, value: Union[str, int], value_type: str = ValueType.value) return self.escape_manager.escape(value, value_type) def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VALUE_TYPE) -> str: - if modifier_function := self.field_value.get(operator.token_type): + if modifier_function := self.modifiers_map.get(operator.token_type): return modifier_function(field, value) raise UnsupportedOperatorException(operator.token_type) +class BaseFieldFieldRender(ABC): + operators_map: ClassVar[dict[str, str]] = {} + + def apply_field_field(self, field_left: str, operator: Identifier, field_right: str) -> str: + if mapped_operator := self.operators_map.get(operator.token_type): + return f"{field_left} {mapped_operator} {field_right}" + raise UnsupportedOperatorException(operator.token_type) + + class QueryRender(ABC): comment_symbol: str = None details: PlatformDetails = None @@ -180,6 +189,13 @@ def render_not_supported_functions(self, not_supported_functions: list) -> str: not_supported_functions_str = "\n".join(line_template + func.lstrip() for func in not_supported_functions) return "\n\n" + self.wrap_with_comment(f"{self.unsupported_functions_text}\n{not_supported_functions_str}") + def wrap_with_not_supported_functions(self, query: str, not_supported_functions: Optional[list] = None) -> str: + if not_supported_functions and wrap_query_with_meta_info_ctx_var.get(): + rendered_not_supported = self.render_not_supported_functions(not_supported_functions) + return query + rendered_not_supported + + return query + def wrap_with_comment(self, value: str) -> str: return f"{self.comment_symbol} {value}" @@ -199,13 +215,14 @@ class PlatformQueryRender(QueryRender): group_token = "(%s)" query_parts_delimiter = " " - field_value_map = BaseQueryFieldValue(or_token=or_token) + field_field_render = BaseFieldFieldRender() + field_value_render = BaseFieldValueRender(or_token=or_token) raw_log_field_pattern_map: ClassVar[dict[str, str]] = None def __init__(self): super().__init__() - self.operator_map = { + self.logical_operators_map = { LogicalOperatorType.AND: f" {self.and_token} ", LogicalOperatorType.OR: f" {self.or_token} ", LogicalOperatorType.NOT: f" {self.not_token} ", @@ -233,31 +250,34 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]: def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): - if token.alias: - field_name = token.alias.name - else: - mapped_fields = self.map_field(token.field, source_mapping) - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value( - field=field, operator=token.operator, value=token.value - ) - for field in mapped_fields - ] - ) - - field_name = mapped_fields[0] - - return self.field_value_map.apply_field_value(field=field_name, operator=token.operator, value=token.value) - + mapped_fields = [token.alias.name] if token.alias else self.map_field(token.field, source_mapping) + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] + ) + return self.group_token % joined if len(mapped_fields) > 1 else joined + if isinstance(token, FieldField): + alias_left, field_left = token.alias_left, token.field_left + mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping) + alias_right, field_right = token.alias_right, token.field_right + mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping) + cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right)) + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_field_render.apply_field_field(pair[0], token.operator, pair[1]) + for pair in cross_paired_fields + ] + ) + return self.group_token % joined if len(cross_paired_fields) > 1 else joined if isinstance(token, Function): func_render = self.platform_functions.manager.get_in_query_render(token.name) return func_render.render(token, source_mapping) if isinstance(token, Keyword): - return self.field_value_map.apply_field_value(field="", operator=token.operator, value=token.value) + return self.field_value_render.apply_field_value(field="", operator=token.operator, value=token.value) if token.token_type in LogicalOperatorType: - return self.operator_map.get(token.token_type) + return self.logical_operators_map.get(token.token_type) return token.token_type @@ -273,8 +293,8 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields)) return "".join(result_values) - def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str: - if meta_info and (meta_info.id or meta_info.title): + def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str: + if wrap_query_with_meta_info_ctx_var.get() and meta_info and (meta_info.id or meta_info.title): meta_info_dict = { "name: ": meta_info.title, "uuid: ": meta_info.id, @@ -307,11 +327,8 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = self._join_query_parts(prefix, query, functions) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) @staticmethod def unique_queries(queries_map: dict[str, str]) -> dict[str, dict[str]]: @@ -342,7 +359,7 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp return source_mappings - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: return self.finalize_query( prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info ) @@ -380,7 +397,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap defined_raw_log_fields.append(prefix) return "\n".join(defined_raw_log_fields) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} errors = [] source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -417,6 +434,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index 45486ef1..7530af5f 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Any, ClassVar, Optional, Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -29,18 +30,18 @@ UnsupportedOperatorException, ) from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function from app.translator.core.models.functions.eval import EvalArg from app.translator.core.models.functions.group_by import GroupByFunction +from app.translator.core.models.functions.join import JoinFunction from app.translator.core.models.functions.rename import RenameArg from app.translator.core.models.functions.sort import SortArg +from app.translator.core.models.functions.union import UnionFunction from app.translator.core.models.identifier import Identifier from app.translator.core.str_value_manager import StrValue, StrValueManager from app.translator.tools.utils import get_match_group -TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field] - class BaseTokenizer(ABC): @abstractmethod @@ -323,13 +324,18 @@ def filter_tokens( ) -> list[TOKEN_TYPE]: return [token for token in tokens if isinstance(token, token_type)] - def get_field_tokens_from_func_args( + def get_field_tokens_from_func_args( # noqa: PLR0912 self, args: list[Union[Field, FieldValue, Keyword, Identifier, Function, SortArg]] ) -> list[Field]: result = [] for arg in args: if isinstance(arg, Field): result.append(arg) + elif isinstance(arg, FieldField): + if not arg.alias_left or arg.alias_left.name != arg.field_left.source_name: + result.append(arg.field_left) + if not arg.alias_right or arg.alias_right.name != arg.field_right.source_name: + result.append(arg.field_right) elif isinstance(arg, FieldValue): if not arg.alias or arg.alias.name != arg.field.source_name: result.append(arg.field) @@ -337,6 +343,8 @@ def get_field_tokens_from_func_args( result.extend(self.get_field_tokens_from_func_args(args=arg.args)) result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_])) + elif isinstance(arg, (JoinFunction, UnionFunction)): + continue elif isinstance(arg, Function): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) elif isinstance(arg, SortArg) and isinstance(arg.field, Field): diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena.py b/uncoder-core/app/translator/platforms/athena/renders/athena.py index a62e5b00..8550c94a 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena.py @@ -21,10 +21,10 @@ from app.translator.managers import render_manager from app.translator.platforms.athena.const import athena_details from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender -class AthenaFieldValue(SqlFieldValue): +class AthenaFieldValueRender(SqlFieldValueRender): details: PlatformDetails = athena_details @@ -35,7 +35,7 @@ class AthenaQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = AthenaFieldValue(or_token=or_token) + field_value_render = AthenaFieldValueRender(or_token=or_token) comment_symbol = "--" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py index 05826d08..6c0c1665 100644 --- a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager -class AQLFieldValue(BaseQueryFieldValue): +class AQLFieldValueRender(BaseFieldValueRender): str_value_manager = aql_str_value_manager @staticmethod @@ -127,8 +127,6 @@ class AQLQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = AQLFieldValue(or_token=or_token) - def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) extra_condition = log_source_signature.extra_condition diff --git a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py index b5994499..f8511d82 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.lucene.mapping import LuceneLogSourceSignature from app.translator.platforms.base.lucene.str_value_manager import lucene_str_value_manager -class LuceneFieldValue(BaseQueryFieldValue): +class LuceneFieldValueRender(BaseFieldValueRender): str_value_manager = lucene_str_value_manager @staticmethod diff --git a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py index b2c12068..74adf32b 100644 --- a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py @@ -21,11 +21,11 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.platforms.base.spl.escape_manager import spl_escape_manager -class SplFieldValue(BaseQueryFieldValue): +class SplFieldValueRender(BaseFieldValueRender): escape_manager = spl_escape_manager def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: diff --git a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py index 43904a1e..d69f1590 100644 --- a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py @@ -22,10 +22,10 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import LogSourceSignature -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender -class SqlFieldValue(BaseQueryFieldValue): +class SqlFieldValueRender(BaseFieldValueRender): def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py index 4101b825..8bcbe56f 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py @@ -23,14 +23,14 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import chronicle_query_details from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings -class ChronicleFieldValue(BaseQueryFieldValue): +class ChronicleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = chronicle_query_details escape_manager = chronicle_escape_manager @@ -109,6 +109,6 @@ class ChronicleQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = ChronicleFieldValue(or_token=or_token) + field_value_render = ChronicleFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py index aaa64384..1961c72b 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py @@ -26,12 +26,12 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details -from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValue, ChronicleQueryRender +from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValueRender, ChronicleQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated Chronicle Security rule." -class ChronicleRuleFieldValue(ChronicleFieldValue): +class ChronicleRuleFieldValueRender(ChronicleFieldValueRender): details: PlatformDetails = chronicle_rule_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -85,7 +85,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: class ChronicleSecurityRuleRender(ChronicleQueryRender): details: PlatformDetails = chronicle_rule_details or_token = "or" - field_value_map = ChronicleRuleFieldValue(or_token=or_token) + field_value_render = ChronicleRuleFieldValueRender(or_token=or_token) @staticmethod def prepare_title(title: str) -> str: diff --git a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py index 8c6630e9..3e5900cc 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings -class CrowdStrikeFieldValue(SplFieldValue): +class CrowdStrikeFieldValueRender(SplFieldValueRender): details = crowdstrike_query_details @@ -36,7 +36,7 @@ class CrowdStrikeQueryRender(SplQueryRender): platform_functions: CrowdStrikeFunctions = None or_token = "OR" - field_value_map = CrowdStrikeFieldValue(or_token=or_token) + field_value_render = CrowdStrikeFieldValueRender(or_token=or_token) comment_symbol = "`" def init_platform_functions(self) -> None: diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py index 09fad79b..0b7b20c4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -50,7 +50,7 @@ class ElasticSearchRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticSearchRuleFieldValue(or_token=or_token) + field_value_render = ElasticSearchRuleFieldValue(or_token=or_token) def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): @@ -109,7 +109,4 @@ def finalize_query( } ) rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py index 104b8ecc..9d7914ab 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py @@ -49,7 +49,7 @@ class ElastAlertRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticAlertRuleFieldValue(or_token=or_token) + field_value_render = ElasticAlertRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -75,7 +75,4 @@ def finalize_query( ) rule = rule.replace("", meta_info.title or _AUTOGENERATED_TEMPLATE) rule = rule.replace("", _SEVERITIES_MAP[meta_info.severity]) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py index 8d2db1d0..2e6a12f0 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -class ElasticSearchFieldValue(LuceneFieldValue): +class ElasticSearchFieldValue(LuceneFieldValueRender): details: PlatformDetails = elasticsearch_lucene_query_details @@ -34,4 +34,4 @@ class ElasticSearchQueryRender(LuceneQueryRender): mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = ElasticSearchFieldValue(or_token=or_token) + field_value_render = ElasticSearchFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py index c3b6a46a..53a4acf5 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py @@ -45,7 +45,7 @@ class KibanaRuleRender(ElasticSearchQueryRender): details: PlatformDetails = kibana_rule_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = KibanaFieldValue(or_token=or_token) + field_value_render = KibanaFieldValue(or_token=or_token) def finalize_query( self, @@ -74,7 +74,4 @@ def finalize_query( references=meta_info.references, ) rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py index 9a013dcf..d8421977 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py @@ -45,7 +45,7 @@ class XPackWatcherRuleRender(ElasticSearchQueryRender): details: PlatformDetails = xpack_watcher_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = XpackWatcherRuleFieldValue(or_token=or_token) + field_value_render = XpackWatcherRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -78,7 +78,4 @@ def finalize_query( rule["input"]["search"]["request"]["indices"] = indices rule["actions"]["send_email"]["email"]["subject"] = meta_info.title or _AUTOGENERATED_TEMPLATE rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index 65ca0b07..dfbc2ee6 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -18,6 +18,7 @@ from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.const import TOKEN_TYPE from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType @@ -28,9 +29,8 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue -from app.translator.core.tokenizer import TOKEN_TYPE from app.translator.managers import render_manager from app.translator.platforms.forti_siem.const import ( FORTI_SIEM_RULE, @@ -76,7 +76,7 @@ ] -class FortiSiemFieldValue(BaseQueryFieldValue): +class FortiSiemFieldValueRender(BaseFieldValueRender): details: PlatformDetails = forti_siem_rule_details str_value_manager = forti_siem_str_value_manager @@ -194,7 +194,7 @@ class FortiSiemRuleRender(PlatformQueryRender): group_token = "(%s)" - field_value_map = FortiSiemFieldValue(or_token=or_token) + field_value_render = FortiSiemFieldValueRender(or_token=or_token) @staticmethod def __is_negated_token(prev_token: TOKEN_TYPE) -> bool: @@ -244,7 +244,7 @@ def __replace_not_tokens(self, tokens: list[TOKEN_TYPE]) -> list[TOKEN_TYPE]: return tokens - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -324,11 +324,7 @@ def finalize_query( rule = rule.replace("", query) rule = rule.replace("", ", ".join(args_list)) rule = rule.replace("", self.get_attr_str(fields.copy())) - - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) @staticmethod def get_attr_str(fields: set[str]) -> str: diff --git a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py index 2bdf001e..986ddd93 100644 --- a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py +++ b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.graylog.const import graylog_details from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings -class GraylogFieldValue(LuceneFieldValue): +class GraylogFieldValue(LuceneFieldValueRender): details: PlatformDetails = graylog_details @@ -34,4 +34,4 @@ class GraylogQueryRender(LuceneQueryRender): mappings: GraylogMappings = graylog_mappings or_token = "OR" - field_value_map = GraylogFieldValue(or_token=or_token) + field_value_render = GraylogFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py index 0348bfb0..3c73c234 100644 --- a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py +++ b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender from app.translator.platforms.hunters.const import hunters_details from app.translator.platforms.hunters.mapping import HuntersMappings, hunters_mappings -class HuntersFieldValue(SqlFieldValue): +class HuntersFieldValueRender(SqlFieldValueRender): details: PlatformDetails = hunters_details @@ -35,7 +35,7 @@ class HuntersQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = HuntersFieldValue(or_token=or_token) + field_value_render = HuntersFieldValueRender(or_token=or_token) @staticmethod def _finalize_search_query(query: str) -> str: diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 624fa3d7..4a288491 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -30,7 +30,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_axon_query_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_query_escape_manager @@ -41,7 +41,7 @@ class LogRhythmRegexRenderException(BaseRenderException): ... -class LogRhythmAxonFieldValue(BaseQueryFieldValue): +class LogRhythmAxonFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logrhythm_axon_query_details escape_manager = logrhythm_query_escape_manager @@ -204,7 +204,7 @@ class LogRhythmAxonQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = LogRhythmAxonFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonFieldValueRender(or_token=or_token) mappings: LogRhythmAxonMappings = logrhythm_axon_mappings comment_symbol = "//" @@ -224,7 +224,7 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp mapped_fields = self.map_field(token.field, source_mapping) except StrictPlatformException: try: - return self.field_value_map.apply_field_value( + return self.field_value_render.apply_field_value( field=UNMAPPED_FIELD_DEFAULT_NAME, operator=token.operator, value=token.value ) except LogRhythmRegexRenderException as exc: @@ -232,20 +232,17 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp f"Uncoder does not support complex regexp for unmapped field:" f" {token.field.source_name} for LogRhythm Axon" ) from exc - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value(field=field, operator=token.operator, value=token.value) - for field in mapped_fields - ] - ) - return self.field_value_map.apply_field_value( - field=mapped_fields[0], operator=token.operator, value=token.value + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] ) + return self.group_token % joined if len(mapped_fields) > 1 else joined return super().apply_token(token, source_mapping) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py index 20514140..2e68c2d1 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py @@ -29,7 +29,7 @@ from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import ( - LogRhythmAxonFieldValue, + LogRhythmAxonFieldValueRender, LogRhythmAxonQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -44,7 +44,7 @@ } -class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): +class LogRhythmAxonRuleFieldValueRender(LogRhythmAxonFieldValueRender): details: PlatformDetails = logrhythm_axon_rule_details escape_manager = logrhythm_rule_escape_manager @@ -53,7 +53,7 @@ class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): class LogRhythmAxonRuleRender(LogRhythmAxonQueryRender): details: PlatformDetails = logrhythm_axon_rule_details or_token = "or" - field_value_map = LogRhythmAxonRuleFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonRuleFieldValueRender(or_token=or_token) def finalize_query( self, @@ -93,7 +93,4 @@ def finalize_query( ] json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py index 9cb7cf05..e1ed4818 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py @@ -23,7 +23,7 @@ from app.translator.core.mapping import SourceMapping from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.escape_manager import logscale_escape_manager @@ -31,7 +31,7 @@ from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings -class LogScaleFieldValue(BaseQueryFieldValue): +class LogScaleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logscale_query_details escape_manager = logscale_escape_manager @@ -102,7 +102,7 @@ class LogScaleQueryRender(PlatformQueryRender): and_token = "" not_token = "not" - field_value_map = LogScaleFieldValue(or_token=or_token) + field_value_render = LogScaleFieldValueRender(or_token=or_token) def init_platform_functions(self) -> None: self.platform_functions = log_scale_functions @@ -123,8 +123,5 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = super().finalize_query(prefix=prefix, query=query, functions=functions) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py index 4b3af0fb..a6628045 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py @@ -26,13 +26,13 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details -from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValue, LogScaleQueryRender +from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValueRender, LogScaleQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Falcon LogScale Alert" -class LogScaleAlertFieldValue(LogScaleFieldValue): +class LogScaleAlertFieldValueRender(LogScaleFieldValueRender): details: PlatformDetails = logscale_alert_details @@ -40,7 +40,7 @@ class LogScaleAlertFieldValue(LogScaleFieldValue): class LogScaleAlertRender(LogScaleQueryRender): details: PlatformDetails = logscale_alert_details or_token = "or" - field_value_map = LogScaleAlertFieldValue(or_token=or_token) + field_value_render = LogScaleAlertFieldValueRender(or_token=or_token) def finalize_query( self, @@ -70,8 +70,5 @@ def finalize_query( mitre_attack=mitre_attack, ) - json_query = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_query + rendered_not_supported - return json_query + rule_str = json.dumps(rule, indent=4, sort_keys=False) + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py index 7b7a3779..38617b55 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py @@ -23,12 +23,12 @@ from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) -class MicrosoftDefenderFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftDefenderFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_defender_details @@ -38,7 +38,7 @@ class MicrosoftDefenderQueryRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_defender_details platform_functions: MicrosoftFunctions = None or_token = "or" - field_value_map = MicrosoftDefenderFieldValue(or_token=or_token) + field_value_render = MicrosoftDefenderFieldValueRender(or_token=or_token) is_strict_mapping = True diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py index 3153f8d4..7ef6f1f9 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py @@ -22,7 +22,7 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.escape_manager import microsoft_escape_manager @@ -30,7 +30,7 @@ from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings -class MicrosoftSentinelFieldValue(BaseQueryFieldValue): +class MicrosoftSentinelFieldValueRender(BaseFieldValueRender): details: PlatformDetails = microsoft_sentinel_query_details escape_manager = microsoft_escape_manager @@ -128,7 +128,7 @@ class MicrosoftSentinelQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = MicrosoftSentinelFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelFieldValueRender(or_token=or_token) mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings comment_symbol = "//" diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index e2fdb81f..b5631ef5 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -28,7 +28,7 @@ from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -42,7 +42,7 @@ } -class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftSentinelRuleFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_sentinel_rule_details @@ -50,7 +50,7 @@ class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): class MicrosoftSentinelRuleRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_sentinel_rule_details or_token = "or" - field_value_map = MicrosoftSentinelRuleFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelRuleFieldValueRender(or_token=or_token) def __create_mitre_threat(self, meta_info: MetaInfoContainer) -> tuple[list, list]: tactics = set() @@ -92,7 +92,4 @@ def finalize_query( rule["tactics"] = mitre_tactics rule["techniques"] = mitre_techniques json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py index 1d2145a7..3298c106 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py @@ -24,12 +24,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.opensearch.const import opensearch_query_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -class OpenSearchFieldValue(LuceneFieldValue): +class OpenSearchFieldValueRender(LuceneFieldValueRender): details: PlatformDetails = opensearch_query_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -102,4 +102,4 @@ class OpenSearchQueryRender(LuceneQueryRender): mappings: OpenSearchMappings = opensearch_mappings or_token = "OR" - field_value_map = OpenSearchFieldValue(or_token=or_token) + field_value_render = OpenSearchFieldValueRender(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py index 3f68e6c6..106a1fe1 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -30,13 +30,13 @@ from app.translator.managers import render_manager from app.translator.platforms.opensearch.const import OPENSEARCH_RULE, opensearch_rule_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValue, OpenSearchQueryRender +from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValueRender, OpenSearchQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated AWS OpenSearch Rule" _SEVERITIES_MAP = {SeverityType.critical: "5", SeverityType.high: "4", SeverityType.medium: "3", SeverityType.low: "2"} -class OpenSearchRuleFieldValue(OpenSearchFieldValue): +class OpenSearchRuleFieldValueRender(OpenSearchFieldValueRender): details: PlatformDetails = opensearch_rule_details @@ -49,7 +49,7 @@ class OpenSearchRuleRender(OpenSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = OpenSearchRuleFieldValue(or_token=or_token) + field_value_render = OpenSearchRuleFieldValueRender(or_token=or_token) def __init__(self): super().__init__() @@ -76,10 +76,7 @@ def finalize_query( rule["triggers"][0]["severity"] = _SEVERITIES_MAP[meta_info.severity] rule["triggers"][0]["actions"][0]["message_template"]["source"] = str(source).replace(", ", ",\n") rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index ba673cd2..bc57adee 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -20,12 +20,14 @@ from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import preset_log_source_str_ctx_var +from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import SourceMapping from app.translator.core.models.field import FieldValue, Keyword from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager from app.translator.platforms.palo_alto.const import cortex_xql_query_details @@ -48,7 +50,8 @@ } -class CortexXQLFieldValue(BaseQueryFieldValue): + +class CortexXQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = cortex_xql_query_details str_value_manager = cortex_xql_str_value_manager @@ -145,6 +148,17 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"_raw_log contains {self._pre_process_value(field ,value, value_type=ValueType.value, wrap_str=True)}" +class CortexXQLFieldFieldRender(BaseFieldFieldRender): + operators_map: ClassVar[dict[str, str]] = { + OperatorType.EQ: "=", + OperatorType.NOT_EQ: "!=", + OperatorType.LT: "<", + OperatorType.LTE: "<=", + OperatorType.GT: ">", + OperatorType.GTE: ">=", + } + + @render_manager.register class CortexXQLQueryRender(PlatformQueryRender): details: PlatformDetails = cortex_xql_query_details @@ -162,7 +176,8 @@ class CortexXQLQueryRender(PlatformQueryRender): not_token = "not" query_parts_delimiter = "\n" - field_value_map = CortexXQLFieldValue(or_token=or_token) + field_field_render = CortexXQLFieldFieldRender() + field_value_render = CortexXQLFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = False @@ -184,7 +199,8 @@ def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, functions_prefix: str = "") -> str: functions_prefix = f"{functions_prefix} | " if functions_prefix else "" - return f"{functions_prefix}{log_source_signature}" + log_source_str = preset_log_source_str_ctx_var.get() or str(log_source_signature) + return f"{functions_prefix}{log_source_str}" def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): diff --git a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py index 0f06fb40..cf4a7d51 100644 --- a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py @@ -19,14 +19,15 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.aql.renders.aql import AQLFieldValue, AQLQueryRender +from app.translator.platforms.base.aql.renders.aql import AQLFieldValueRender, AQLQueryRender from app.translator.platforms.qradar.const import qradar_query_details -class QradarFieldValue(AQLFieldValue): +class QradarFieldValueRender(AQLFieldValueRender): details: PlatformDetails = qradar_query_details @render_manager.register class QradarQueryRender(AQLQueryRender): details: PlatformDetails = qradar_query_details + field_value_render = QradarFieldValueRender(or_token=AQLQueryRender.or_token) diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index dc33a507..856fd4a3 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -281,10 +281,10 @@ def __get_source_mapping(self, source_mapping_ids: list[str]) -> SourceMapping: return self.mappings.get_source_mapping(DEFAULT_MAPPING_NAME) - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: raise NotImplementedError - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: self.reset_counters() meta_info = query_container.meta_info @@ -316,6 +316,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py index f9404cac..e14c6bfc 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings -class SplunkFieldValue(SplFieldValue): +class SplunkFieldValueRender(SplFieldValueRender): details: PlatformDetails = splunk_query_details @@ -35,7 +35,7 @@ class SplunkQueryRender(SplQueryRender): or_token = "OR" - field_value_map = SplunkFieldValue(or_token=or_token) + field_value_render = SplunkFieldValueRender(or_token=or_token) mappings: SplunkMappings = splunk_mappings platform_functions: SplunkFunctions = None diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py index ef0d097d..5dc2096a 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py @@ -25,14 +25,14 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details -from app.translator.platforms.splunk.renders.splunk import SplunkFieldValue, SplunkQueryRender +from app.translator.platforms.splunk.renders.splunk import SplunkFieldValueRender, SplunkQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Splunk Alert" _SEVERITIES_MAP = {SeverityType.critical: "4", SeverityType.high: "3", SeverityType.medium: "2", SeverityType.low: "1"} -class SplunkAlertFieldValue(SplunkFieldValue): +class SplunkAlertFieldValueRender(SplunkFieldValueRender): details: PlatformDetails = splunk_alert_details @@ -40,7 +40,7 @@ class SplunkAlertFieldValue(SplunkFieldValue): class SplunkAlertRender(SplunkQueryRender): details: PlatformDetails = splunk_alert_details or_token = "OR" - field_value_map = SplunkAlertFieldValue(or_token=or_token) + field_value_render = SplunkAlertFieldValueRender(or_token=or_token) @staticmethod def __create_mitre_threat(meta_info: MetaInfoContainer) -> dict: @@ -74,7 +74,4 @@ def finalize_query( if mitre_techniques: mitre_str = f"action.correlationsearch.annotations = {mitre_techniques})" rule = rule.replace("", mitre_str) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy