From 002430c508746ad7e785500cce30a2cfd2084088 Mon Sep 17 00:00:00 2001 From: "oleksandr.volha" Date: Fri, 13 Sep 2024 11:35:08 +0300 Subject: [PATCH] anomali render --- uncoder-core/app/translator/const.py | 2 +- uncoder-core/app/translator/core/render.py | 2 +- .../app/translator/core/str_value_manager.py | 19 ++++ .../mappings/platforms/anomali/common.yml | 31 ++++++ .../mappings/platforms/anomali/default.yml | 5 + .../translator/platforms/anomali/__init__.py | 1 + .../app/translator/platforms/anomali/const.py | 11 ++ .../translator/platforms/anomali/mapping.py | 18 ++++ .../platforms/anomali/renders/__init__.py | 0 .../platforms/anomali/renders/anomali.py | 100 ++++++++++++++++++ .../platforms/base/aql/str_value_manager.py | 33 +----- .../platforms/base/sql/escape_manager.py | 18 ++++ .../platforms/base/sql/str_value_manager.py | 100 ++++++++++++++++++ .../platforms/sigma/str_value_manager.py | 36 +------ uncoder-core/app/translator/translator.py | 20 ++-- 15 files changed, 320 insertions(+), 76 deletions(-) create mode 100644 uncoder-core/app/translator/mappings/platforms/anomali/common.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/anomali/default.yml create mode 100644 uncoder-core/app/translator/platforms/anomali/__init__.py create mode 100644 uncoder-core/app/translator/platforms/anomali/const.py create mode 100644 uncoder-core/app/translator/platforms/anomali/mapping.py create mode 100644 uncoder-core/app/translator/platforms/anomali/renders/__init__.py create mode 100644 uncoder-core/app/translator/platforms/anomali/renders/anomali.py create mode 100644 uncoder-core/app/translator/platforms/base/sql/escape_manager.py create mode 100644 uncoder-core/app/translator/platforms/base/sql/str_value_manager.py diff --git a/uncoder-core/app/translator/const.py b/uncoder-core/app/translator/const.py index 6db1167e..767fe882 100644 --- a/uncoder-core/app/translator/const.py +++ b/uncoder-core/app/translator/const.py @@ -9,4 +9,4 @@ CTI_IOCS_PER_QUERY_LIMIT = 25 -DEFAULT_VALUE_TYPE = Union[int, str, StrValue, list[Union[int, str, StrValue]]] +DEFAULT_VALUE_TYPE = Union[bool, int, str, StrValue, list[Union[int, str, StrValue]]] diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 8e9f8373..778fbfb2 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -90,7 +90,7 @@ def _map_bool_value(value: bool) -> str: def _pre_process_value( self, field: str, - value: Union[int, str, StrValue], + value: Union[bool, int, str, StrValue], value_type: str = ValueType.value, wrap_str: bool = False, wrap_int: bool = False, diff --git a/uncoder-core/app/translator/core/str_value_manager.py b/uncoder-core/app/translator/core/str_value_manager.py index b5718e3a..5ee02312 100644 --- a/uncoder-core/app/translator/core/str_value_manager.py +++ b/uncoder-core/app/translator/core/str_value_manager.py @@ -130,6 +130,25 @@ def has_spec_symbols(self) -> bool: return any(isinstance(el, BaseSpecSymbol) for el in self.split_value) +RE_STR_SPEC_SYMBOLS_MAP = { + "?": ReZeroOrOneQuantifier, + "*": ReZeroOrMoreQuantifier, + "+": ReOneOrMoreQuantifier, + "^": ReCaretSymbol, + "$": ReEndOfStrSymbol, + ".": ReAnySymbol, + "[": ReLeftSquareBracket, + "]": ReRightSquareBracket, + "(": ReLeftParenthesis, + ")": ReRightParenthesis, + "{": ReLeftCurlyBracket, + "}": ReRightCurlyBracket, + "|": ReOrOperator, + ",": ReCommaSymbol, + "-": ReHyphenSymbol, +} + + CONTAINER_SPEC_SYMBOLS_MAP = { SingleSymbolWildCard: "?", UnboundLenWildCard: "*", diff --git a/uncoder-core/app/translator/mappings/platforms/anomali/common.yml b/uncoder-core/app/translator/mappings/platforms/anomali/common.yml new file mode 100644 index 00000000..7b069f4c --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/anomali/common.yml @@ -0,0 +1,31 @@ +platform: Anomali +description: Common field mapping + +field_mapping: + c-uri-query: url + c-useragent: user_agent + CommandLine: command_line + DestinationHostname: dest + DestinationIp: dest_ip + DestinationPort: dest_port + Details: reg_value_data + dst_ip: dest_ip + dst_port: dest_port + EventID: event_id + EventName: event_name + FileName: file_name + FilePath: file_path + Image: image + NewProcessName: image + OriginalFileName: original_file_name + ParentCommandLine: parent_command_line + ParentImage: parent_image + ParentProcessID: parent_process_id + Platform: platform + ProcessCommandLine: command_line + ProcessID: process_id + SourceImage: parent_image + SourcePort: src_port + TargetFilename: file_name + TargetObject: reg_key + UserAgent: user_agent diff --git a/uncoder-core/app/translator/mappings/platforms/anomali/default.yml b/uncoder-core/app/translator/mappings/platforms/anomali/default.yml new file mode 100644 index 00000000..fed2954e --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/anomali/default.yml @@ -0,0 +1,5 @@ +platform: Anomali +source: default + + +default_log_source: {} diff --git a/uncoder-core/app/translator/platforms/anomali/__init__.py b/uncoder-core/app/translator/platforms/anomali/__init__.py new file mode 100644 index 00000000..5cd64d01 --- /dev/null +++ b/uncoder-core/app/translator/platforms/anomali/__init__.py @@ -0,0 +1 @@ +from app.translator.platforms.anomali.renders.anomali import AnomaliQueryRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/anomali/const.py b/uncoder-core/app/translator/platforms/anomali/const.py new file mode 100644 index 00000000..3d14733d --- /dev/null +++ b/uncoder-core/app/translator/platforms/anomali/const.py @@ -0,0 +1,11 @@ +from app.translator.core.models.platform_details import PlatformDetails + +ANOMALI_QUERY_DETAILS = { + "platform_id": "anomali-aql-query", + "name": "Anomali Security Analytics Query", + "group_name": "Anomali Security Analytics", + "platform_name": "Query", + "group_id": "anomali", +} + +anomali_query_details = PlatformDetails(**ANOMALI_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/anomali/mapping.py b/uncoder-core/app/translator/platforms/anomali/mapping.py new file mode 100644 index 00000000..5c7e13a3 --- /dev/null +++ b/uncoder-core/app/translator/platforms/anomali/mapping.py @@ -0,0 +1,18 @@ +from app.translator.core.mapping import BaseCommonPlatformMappings, LogSourceSignature +from app.translator.platforms.anomali.const import anomali_query_details + + +class AnomaliLogSourceSignature(LogSourceSignature): + def is_suitable(self) -> bool: + return True + + def __str__(self) -> str: + return "" + + +class AnomaliMappings(BaseCommonPlatformMappings): + def prepare_log_source_signature(self, mapping: dict) -> AnomaliLogSourceSignature: # noqa: ARG002 + return AnomaliLogSourceSignature() + + +anomali_query_mappings = AnomaliMappings(platform_dir="anomali", platform_details=anomali_query_details) diff --git a/uncoder-core/app/translator/platforms/anomali/renders/__init__.py b/uncoder-core/app/translator/platforms/anomali/renders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uncoder-core/app/translator/platforms/anomali/renders/anomali.py b/uncoder-core/app/translator/platforms/anomali/renders/anomali.py new file mode 100644 index 00000000..1da26ab7 --- /dev/null +++ b/uncoder-core/app/translator/platforms/anomali/renders/anomali.py @@ -0,0 +1,100 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.custom_types.values import ValueType +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.managers import render_manager +from app.translator.platforms.anomali.const import anomali_query_details +from app.translator.platforms.anomali.mapping import AnomaliMappings, anomali_query_mappings +from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager + + +class AnomaliFieldValueRender(BaseFieldValueRender): + details: PlatformDetails = anomali_query_details + str_value_manager = sql_str_value_manager + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f"'{value}'" + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" + return f"{field} = {self._pre_process_value(field, value, wrap_str=True)}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" + return f"{field} != {self._pre_process_value(field, value, wrap_str=True)}" + + def less_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} < {self._pre_process_value(field, value, wrap_str=True)}" + + def less_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} <= {self._pre_process_value(field, value, wrap_str=True)}" + + def greater_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} > {self._pre_process_value(field, value, wrap_str=True)}" + + def greater_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} >= {self._pre_process_value(field, value, wrap_str=True)}" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" + return f"{field} like '%{self._pre_process_value(field, value)}%'" + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" + return f"{field} like '%{self._pre_process_value(field, value)}'" + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" + return f"{field} like '{self._pre_process_value(field, value)}%'" + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" + regex_str = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=True) + return f"regexp_like({field}, {regex_str})" + + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f'message contains "{self._pre_process_value(field, value)}"' + + +@render_manager.register +class AnomaliQueryRender(PlatformQueryRender): + details: PlatformDetails = anomali_query_details + mappings: AnomaliMappings = anomali_query_mappings + + or_token = "OR" + and_token = "AND" + not_token = "NOT" + + comment_symbol = "--" + is_single_line_comment = True + + field_value_render = AnomaliFieldValueRender(or_token=or_token) + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"| where {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py index 2e189db0..6c2a071b 100644 --- a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py @@ -23,26 +23,12 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( CONTAINER_SPEC_SYMBOLS_MAP, + RE_STR_SPEC_SYMBOLS_MAP, BaseSpecSymbol, - ReAnySymbol, - ReCaretSymbol, - ReCommaSymbol, ReDigitalSymbol, - ReEndOfStrSymbol, - ReHyphenSymbol, - ReLeftCurlyBracket, - ReLeftParenthesis, - ReLeftSquareBracket, - ReOneOrMoreQuantifier, - ReOrOperator, - ReRightCurlyBracket, - ReRightParenthesis, - ReRightSquareBracket, ReWhiteSpaceSymbol, ReWordBoundarySymbol, ReWordSymbol, - ReZeroOrMoreQuantifier, - ReZeroOrOneQuantifier, SingleSymbolWildCard, StrValue, StrValueManager, @@ -50,23 +36,6 @@ ) from app.translator.platforms.base.aql.escape_manager import aql_escape_manager -RE_STR_SPEC_SYMBOLS_MAP = { - "?": ReZeroOrOneQuantifier, - "*": ReZeroOrMoreQuantifier, - "+": ReOneOrMoreQuantifier, - "^": ReCaretSymbol, - "$": ReEndOfStrSymbol, - ".": ReAnySymbol, - "[": ReLeftSquareBracket, - "]": ReRightSquareBracket, - "(": ReLeftParenthesis, - ")": ReRightParenthesis, - "{": ReLeftCurlyBracket, - "}": ReRightCurlyBracket, - "|": ReOrOperator, - ",": ReCommaSymbol, - "-": ReHyphenSymbol, -} AQL_CONTAINER_SPEC_SYMBOLS_MAP = copy.copy(CONTAINER_SPEC_SYMBOLS_MAP) AQL_CONTAINER_SPEC_SYMBOLS_MAP.update({SingleSymbolWildCard: "_", UnboundLenWildCard: "%"}) diff --git a/uncoder-core/app/translator/platforms/base/sql/escape_manager.py b/uncoder-core/app/translator/platforms/base/sql/escape_manager.py new file mode 100644 index 00000000..8e5be92a --- /dev/null +++ b/uncoder-core/app/translator/platforms/base/sql/escape_manager.py @@ -0,0 +1,18 @@ +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.escape_manager import EscapeManager +from app.translator.core.models.escape_details import EscapeDetails + + +class SQLEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern=r"(')", escape_symbols=r"'\1")], + ValueType.regex_value: [ + EscapeDetails(pattern=r"([$^*+()\[\]{}|.?\-\\])", escape_symbols=r"\\\1"), + EscapeDetails(pattern=r"(')", escape_symbols=r"'\1"), + ], + } + + +sql_escape_manager = SQLEscapeManager() diff --git a/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py new file mode 100644 index 00000000..0c2f03ba --- /dev/null +++ b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py @@ -0,0 +1,100 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" + +import copy +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.str_value_manager import ( + CONTAINER_SPEC_SYMBOLS_MAP, + RE_STR_SPEC_SYMBOLS_MAP, + BaseSpecSymbol, + ReDigitalSymbol, + ReWhiteSpaceSymbol, + ReWordBoundarySymbol, + ReWordSymbol, + SingleSymbolWildCard, + StrValue, + StrValueManager, + UnboundLenWildCard, +) +from app.translator.platforms.base.sql.escape_manager import sql_escape_manager + +SQL_CONTAINER_SPEC_SYMBOLS_MAP = copy.copy(CONTAINER_SPEC_SYMBOLS_MAP) +SQL_CONTAINER_SPEC_SYMBOLS_MAP.update({SingleSymbolWildCard: "_", UnboundLenWildCard: "%"}) + + +class SQLStrValueManager(StrValueManager): + escape_manager = sql_escape_manager + container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = SQL_CONTAINER_SPEC_SYMBOLS_MAP + re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { + "b": ReWordBoundarySymbol, + "w": ReWordSymbol, + "d": ReDigitalSymbol, + "s": ReWhiteSpaceSymbol, + } + re_str_spec_symbols_map = RE_STR_SPEC_SYMBOLS_MAP + str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { + "_": SingleSymbolWildCard, + "%": UnboundLenWildCard, + } + + def from_str_to_container(self, value: str) -> StrValue: + split = [] + prev_char = None + for char in value: + if char in self.str_spec_symbols_map: + split.append(self.str_spec_symbols_map[char]()) + else: + if char == "'": + if prev_char == "'": + split.append(char) + prev_char = char + continue + split.append(char) + + prev_char = char + + return StrValue(value, self._concat(split)) + + def from_re_str_to_container(self, value: str) -> StrValue: + value = value.replace("''", "'") + return super().from_re_str_to_container(value) + + def from_container_to_str(self, container: StrValue, value_type: str = ValueType.value) -> str: + result = "" + for el in container.split_value: + if isinstance(el, str): + result += self.escape_manager.escape(el, value_type) + elif isinstance(el, BaseSpecSymbol): + if value_type == ValueType.regex_value: + if isinstance(el, SingleSymbolWildCard): + result += "." + continue + if isinstance(el, UnboundLenWildCard): + result += ".*" + continue + + if pattern := self.container_spec_symbols_map.get(type(el)): + result += pattern + + return result + + +sql_str_value_manager = SQLStrValueManager() diff --git a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py index ae5120df..751db716 100644 --- a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py @@ -18,50 +18,18 @@ """ from app.translator.core.str_value_manager import ( - ReAnySymbol, - ReCaretSymbol, - ReCommaSymbol, + RE_STR_SPEC_SYMBOLS_MAP, ReDigitalSymbol, - ReEndOfStrSymbol, - ReHyphenSymbol, - ReLeftCurlyBracket, - ReLeftParenthesis, - ReLeftSquareBracket, - ReOneOrMoreQuantifier, - ReOrOperator, - ReRightCurlyBracket, - ReRightParenthesis, - ReRightSquareBracket, ReWhiteSpaceSymbol, ReWordBoundarySymbol, ReWordSymbol, - ReZeroOrMoreQuantifier, - ReZeroOrOneQuantifier, SingleSymbolWildCard, StrValue, StrValueManager, - UnboundLenWildCard, + UnboundLenWildCard ) from app.translator.platforms.sigma.escape_manager import sigma_escape_manager -RE_STR_SPEC_SYMBOLS_MAP = { - "?": ReZeroOrOneQuantifier, - "*": ReZeroOrMoreQuantifier, - "+": ReOneOrMoreQuantifier, - "^": ReCaretSymbol, - "$": ReEndOfStrSymbol, - ".": ReAnySymbol, - "[": ReLeftSquareBracket, - "]": ReRightSquareBracket, - "(": ReLeftParenthesis, - ")": ReRightParenthesis, - "{": ReLeftCurlyBracket, - "}": ReRightCurlyBracket, - "|": ReOrOperator, - ",": ReCommaSymbol, - "-": ReHyphenSymbol, -} - class SigmaStrValueManager(StrValueManager): escape_manager = sigma_escape_manager diff --git a/uncoder-core/app/translator/translator.py b/uncoder-core/app/translator/translator.py index a62f870d..15cf428d 100644 --- a/uncoder-core/app/translator/translator.py +++ b/uncoder-core/app/translator/translator.py @@ -1,14 +1,12 @@ import logging -from typing import Optional, Union +from typing import Optional from app.translator.core.exceptions.core import UnsupportedPlatform from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer -from app.translator.core.parser import PlatformQueryParser +from app.translator.core.parser import QueryParser from app.translator.core.render import QueryRender from app.translator.managers import ParserManager, RenderManager, parser_manager, render_manager from app.translator.platforms.elasticsearch.const import ELASTIC_QUERY_TYPES -from app.translator.platforms.roota.parsers.roota import RootAParser -from app.translator.platforms.sigma.parsers.sigma import SigmaParser from app.translator.tools.decorators import handle_translation_exceptions @@ -19,7 +17,7 @@ class Translator: def __init__(self): self.logger = logging.getLogger("translator") - def __get_parser(self, source: str) -> Union[PlatformQueryParser, RootAParser, SigmaParser]: + def __get_parser(self, source: str) -> QueryParser: parser = self.parser_manager.get(source) if not parser: raise UnsupportedPlatform(platform=source, is_parser=True) @@ -41,13 +39,16 @@ def __is_one_vendor_translation(source: str, target: str) -> bool: return False + def parse_raw_query(self, text: str, source: str) -> tuple[QueryParser, RawQueryContainer]: + parser = self.__get_parser(source) + text = parser.remove_comments(text) + return parser, parser.parse_raw_query(text, language=source) + @handle_translation_exceptions def __parse_incoming_data( self, text: str, source: str, target: Optional[str] = None ) -> tuple[RawQueryContainer, Optional[TokenizedQueryContainer]]: - parser = self.__get_parser(source) - text = parser.remove_comments(text) - raw_query_container = parser.parse_raw_query(text, language=source) + parser, raw_query_container = self.parse_raw_query(text=text, source=source) tokenized_query_container = None if not (target and self.__is_one_vendor_translation(raw_query_container.language, target)): tokenized_query_container = parser.parse(raw_query_container) @@ -117,3 +118,6 @@ def get_parsers(self) -> list: def get_renders(self) -> list: return self.render_manager.get_platforms_details + + +app_translator = Translator() pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy