From 2bc9ab20670d88f3f6097169b22058d29c447444 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Fri, 11 Oct 2024 16:38:24 +0300 Subject: [PATCH 1/8] sql str value manager --- .../platforms/anomali/renders/anomali.py | 55 +------------------ .../platforms/base/spl/renders/spl.py | 7 ++- .../platforms/base/sql/renders/sql.py | 45 ++++++++------- .../platforms/base/sql/tokenizer.py | 5 +- 4 files changed, 39 insertions(+), 73 deletions(-) diff --git a/uncoder-core/app/translator/platforms/anomali/renders/anomali.py b/uncoder-core/app/translator/platforms/anomali/renders/anomali.py index 1da26ab7..531cc739 100644 --- a/uncoder-core/app/translator/platforms/anomali/renders/anomali.py +++ b/uncoder-core/app/translator/platforms/anomali/renders/anomali.py @@ -17,65 +17,16 @@ ----------------------------------------------------------------- """ from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.custom_types.values import ValueType from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.core.render import PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.anomali.const import anomali_query_details from app.translator.platforms.anomali.mapping import AnomaliMappings, anomali_query_mappings -from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender -class AnomaliFieldValueRender(BaseFieldValueRender): +class AnomaliFieldValueRender(SqlFieldValueRender): details: PlatformDetails = anomali_query_details - str_value_manager = sql_str_value_manager - - @staticmethod - def _wrap_str_value(value: str) -> str: - return f"'{value}'" - - def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if isinstance(value, list): - return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" - return f"{field} = {self._pre_process_value(field, value, wrap_str=True)}" - - def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if isinstance(value, list): - return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" - return f"{field} != {self._pre_process_value(field, value, wrap_str=True)}" - - def less_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - return f"{field} < {self._pre_process_value(field, value, wrap_str=True)}" - - def less_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - return f"{field} <= {self._pre_process_value(field, value, wrap_str=True)}" - - def greater_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - return f"{field} > {self._pre_process_value(field, value, wrap_str=True)}" - - def greater_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - return f"{field} >= {self._pre_process_value(field, value, wrap_str=True)}" - - def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if isinstance(value, list): - return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" - return f"{field} like '%{self._pre_process_value(field, value)}%'" - - def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if isinstance(value, list): - return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f"{field} like '%{self._pre_process_value(field, value)}'" - - def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if isinstance(value, list): - return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return f"{field} like '{self._pre_process_value(field, value)}%'" - - def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if isinstance(value, list): - return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" - regex_str = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=True) - return f"regexp_like({field}, {regex_str})" def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f'message contains "{self._pre_process_value(field, value)}"' diff --git a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py index c8dffa70..c3c36675 100644 --- a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py @@ -34,7 +34,12 @@ def _wrap_str_value(value: str) -> str: return f'"{value}"' def _pre_process_value( - self, field: str, value: Union[int, str, StrValue], value_type: str = ValueType.value, wrap_str: bool = False + self, + field: str, + value: Union[bool, int, str, StrValue], + value_type: str = ValueType.value, + wrap_str: bool = False, + wrap_int: bool = False, # noqa: ARG002 ) -> Union[int, str]: value = super()._pre_process_value(field, value, value_type=value_type, wrap_str=wrap_str) return self._wrap_str_value(str(value)) if not isinstance(value, str) else value diff --git a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py index 9426c0cc..91abb7a9 100644 --- a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py @@ -17,55 +17,62 @@ ----------------------------------------------------------------- """ -from typing import Union - from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import LogSourceSignature from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager class SqlFieldValueRender(BaseFieldValueRender): + str_value_manager = sql_str_value_manager + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f"'{value}'" + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" - return f"{field} = '{value}'" + return f"{field} = {self._pre_process_value(field, value, wrap_str=True)}" - def less_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} < '{value}'" + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" + return f"{field} != {self._pre_process_value(field, value, wrap_str=True)}" - def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} <= '{value}'" + def less_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} < {self._pre_process_value(field, value, wrap_str=True)}" - def greater_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} > '{value}'" + def less_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} <= {self._pre_process_value(field, value, wrap_str=True)}" - def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} >= '{value}'" + def greater_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} > {self._pre_process_value(field, value, wrap_str=True)}" - def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if isinstance(value, list): - return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" - return f"{field} != '{value}'" + def greater_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return f"{field} >= {self._pre_process_value(field, value, wrap_str=True)}" def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" - return f"{field} ILIKE '%{value}%' ESCAPE '\\'" + return f"{field} like '%{self._pre_process_value(field, value)}%'" def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f"{field} ILIKE '%{value}' ESCAPE '\\'" + return f"{field} like '%{self._pre_process_value(field, value)}'" def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return f"{field} ILIKE '{value}%' ESCAPE '\\'" + return f"{field} like '{self._pre_process_value(field, value)}%'" def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" - return f"{field} ILIKE '{value}' ESCAPE '\\'" + regex_str = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=True) + return f"regexp_like({field}, {regex_str})" class SqlQueryRender(PlatformQueryRender): diff --git a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py index 8292ca14..202ad87d 100644 --- a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py @@ -24,6 +24,7 @@ from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer +from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager from app.translator.tools.utils import get_match_group @@ -51,6 +52,8 @@ class SqlTokenizer(QueryTokenizer): wildcard_symbol = "%" + str_value_manager = sql_str_value_manager + @staticmethod def should_process_value_wildcards(operator: Optional[str]) -> bool: return operator and operator.lower() in ("like",) @@ -65,7 +68,7 @@ def get_operator_and_value( return mapped_operator, bool_value if (s_q_value := get_match_group(match, group_name=ValueType.single_quotes_value)) is not None: - return mapped_operator, s_q_value + return mapped_operator, self.str_value_manager.from_str_to_container(s_q_value) return super().get_operator_and_value(match, mapped_operator, operator) From 26786461f97b610529d3b270f677cc30908f6721 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Thu, 17 Oct 2024 12:06:25 +0300 Subject: [PATCH 2/8] fix --- .../app/translator/platforms/base/sql/str_value_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py index 0c2f03ba..e54e850a 100644 --- a/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py @@ -65,6 +65,8 @@ def from_str_to_container(self, value: str) -> StrValue: if char == "'": if prev_char == "'": split.append(char) + prev_char = None + continue prev_char = char continue split.append(char) From dcf1125f213d0e56ed6c5928a275f785825778a6 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Fri, 18 Oct 2024 10:59:49 +0300 Subject: [PATCH 3/8] parse regex --- .../platforms/base/sql/tokenizer.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py index 202ad87d..2a19205c 100644 --- a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py @@ -22,7 +22,9 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager from app.translator.tools.utils import get_match_group @@ -49,6 +51,7 @@ class SqlTokenizer(QueryTokenizer): ) _value_pattern = rf"{num_value_pattern}|{bool_value_pattern}|{single_quotes_value_pattern}" multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>\d+(?:,\s*\d+)*|'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*'(?:,\s*'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*')*)\)""" # noqa: E501 + re_field_value_pattern = rf"""regexp_like\({field_pattern},\s*'(?P<{ValueType.regex_value}>(?:[:a-zA-Z\*\?0-9=+%#№;\-_,"\.$&^@!\{{\}}\[\]\s?<>|]|\\\'|\\)+)'\)""" # noqa: E501 wildcard_symbol = "%" @@ -77,6 +80,22 @@ def create_field_value(field_name: str, operator: Identifier, value: Union[str, field_name = field_name.strip('"') return FieldValue(source_name=field_name, operator=operator, value=value) + def _search_re_field_value(self, query: str) -> Optional[tuple[FieldValue, str]]: + if match := re.match(self.re_field_value_pattern, query, re.IGNORECASE): + group_dict = match.groupdict() + field_name = group_dict["field_name"] + value = self.str_value_manager.from_re_str_to_container(group_dict[ValueType.regex_value]) + operator = Identifier(token_type=OperatorType.REGEX) + return self.create_field_value(field_name, operator, value), query[match.end() :] + def tokenize(self, query: str) -> list: query = re.sub(r"\s*ESCAPE\s*'.'", "", query) # remove `ESCAPE 'escape_char'` in LIKE expr return super().tokenize(query) + + def _get_next_token( + self, query: str + ) -> tuple[Union[FieldValue, FunctionValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]: + query = query.strip("\n").strip(" ").strip("\n") + if search_result := self._search_re_field_value(query): + return search_result + return super()._get_next_token(query) From 0cd14cb6b2874203c07b6a4b4e3f3588094ce7ff Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Fri, 18 Oct 2024 12:13:18 +0300 Subject: [PATCH 4/8] fix --- .../translator/platforms/sigma/escape_manager.py | 3 ++- .../translator/platforms/sigma/renders/sigma.py | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/uncoder-core/app/translator/platforms/sigma/escape_manager.py b/uncoder-core/app/translator/platforms/sigma/escape_manager.py index c0efb332..26df6163 100644 --- a/uncoder-core/app/translator/platforms/sigma/escape_manager.py +++ b/uncoder-core/app/translator/platforms/sigma/escape_manager.py @@ -7,7 +7,8 @@ class SigmaEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { - ValueType.value: [EscapeDetails(pattern=r"([*?\\])", escape_symbols=r"\\\1")] + ValueType.value: [EscapeDetails(pattern=r"([*?\\])", escape_symbols=r"\\\1")], + ValueType.regex_value: [EscapeDetails(pattern=r"([$^*+()\[\]{}|.?\-\\])", escape_symbols=r"\\\1")] } diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index 51b1b642..e20b7682 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -23,6 +23,7 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import OperatorType +from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.models.query_tokens.keyword import Keyword @@ -211,18 +212,26 @@ def generate_field(self, data: FieldValue, source_mapping: SourceMapping): ): field_name = f"{field_name}|{data.operator.token_type}" - values = self.__pre_process_values(data.values) + value_type_map = { + OperatorType.REGEX: ValueType.regex_value + } + value_type = value_type_map.get(data.operator.token_type, ValueType.value) + values = self.__pre_process_values(data.values, value_type) if len(values) == 1: return {field_name: values[0]} elif len(values) == 0: return {field_name: ""} return {field_name: values} - def __pre_process_values(self, values: DEFAULT_VALUE_TYPE) -> list[Union[int, str]]: + def __pre_process_values( + self, + values: DEFAULT_VALUE_TYPE, + value_type: str = ValueType.value + ) -> list[Union[int, str]]: processed = [] for v in values: if isinstance(v, StrValue): - processed.append(self.str_value_manager.from_container_to_str(v)) + processed.append(self.str_value_manager.from_container_to_str(v, value_type=value_type)) elif isinstance(v, str): processed.append(v) else: From 40d51c4064f51cde1eeea0a781782365de33889c Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Sat, 19 Oct 2024 11:53:19 +0300 Subject: [PATCH 5/8] parse sql escape symbol --- .../app/translator/core/str_value_manager.py | 2 +- uncoder-core/app/translator/core/tokenizer.py | 3 +- .../platforms/base/aql/str_value_manager.py | 4 +-- .../base/lucene/str_value_manager.py | 4 +-- .../platforms/base/spl/str_value_manager.py | 4 +-- .../base/sql/custom_types/__init__.py | 0 .../platforms/base/sql/custom_types/values.py | 5 ++++ .../platforms/base/sql/escape_manager.py | 7 +++-- .../platforms/base/sql/renders/sql.py | 13 ++++++-- .../platforms/base/sql/str_value_manager.py | 30 ++++++++++++------- .../platforms/base/sql/tokenizer.py | 13 ++++---- .../elasticsearch/str_value_manager.py | 17 +++++++++-- .../platforms/sigma/str_value_manager.py | 3 +- 13 files changed, 67 insertions(+), 38 deletions(-) create mode 100644 uncoder-core/app/translator/platforms/base/sql/custom_types/__init__.py create mode 100644 uncoder-core/app/translator/platforms/base/sql/custom_types/values.py diff --git a/uncoder-core/app/translator/core/str_value_manager.py b/uncoder-core/app/translator/core/str_value_manager.py index 5ee02312..9ba3315d 100644 --- a/uncoder-core/app/translator/core/str_value_manager.py +++ b/uncoder-core/app/translator/core/str_value_manager.py @@ -182,7 +182,7 @@ class StrValueManager: container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = CONTAINER_SPEC_SYMBOLS_MAP @staticmethod - def from_str_to_container(value: str) -> StrValue: + def from_str_to_container(value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG004 return StrValue(value=value, split_value=[value]) def from_re_str_to_container(self, value: str) -> StrValue: diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index 5273829c..9a963473 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -162,8 +162,7 @@ def search_multi_value( def _get_field_value_match(self, query: str, operator: str, field_name: str, value_pattern: str) -> re.Match: field_value_pattern = self.get_field_value_pattern(operator, field_name, value_pattern) - field_value_regex = re.compile(field_value_pattern, re.IGNORECASE) - field_value_match = re.match(field_value_regex, query) + field_value_match = re.match(field_value_pattern, query, re.IGNORECASE) if field_value_match is None: raise TokenizerGeneralException(error=f"Value couldn't be found in query part: {query}") diff --git a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py index 6c2a071b..cc6ccb14 100644 --- a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py @@ -18,7 +18,7 @@ """ import copy -from typing import ClassVar +from typing import ClassVar, Optional from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( @@ -55,7 +55,7 @@ class AQLStrValueManager(StrValueManager): "%": UnboundLenWildCard, } - def from_str_to_container(self, value: str) -> StrValue: + def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 split = [] prev_char = None for char in value: diff --git a/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py b/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py index 9eb8e6bc..4f897216 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py @@ -17,7 +17,7 @@ ----------------------------------------------------------------- """ -from typing import ClassVar +from typing import ClassVar, Optional from app.translator.core.str_value_manager import ( BaseSpecSymbol, @@ -68,7 +68,7 @@ class LuceneStrValueManager(StrValueManager): } re_str_spec_symbols_map = RE_STR_SPEC_SYMBOLS_MAP - def from_str_to_container(self, value: str) -> StrValue: + def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 split = [] prev_char = None for char in value: diff --git a/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py b/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py index 84ebaab7..0059e8b3 100644 --- a/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py @@ -16,7 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ -from typing import ClassVar +from typing import ClassVar, Optional from app.translator.core.str_value_manager import BaseSpecSymbol, StrValue, StrValueManager, UnboundLenWildCard from app.translator.platforms.base.spl.escape_manager import spl_escape_manager @@ -26,7 +26,7 @@ class SplStrValueManager(StrValueManager): escape_manager = spl_escape_manager str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": UnboundLenWildCard} - def from_str_to_container(self, value: str) -> StrValue: + def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 split = [] prev_char = None for char in value: diff --git a/uncoder-core/app/translator/platforms/base/sql/custom_types/__init__.py b/uncoder-core/app/translator/platforms/base/sql/custom_types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uncoder-core/app/translator/platforms/base/sql/custom_types/values.py b/uncoder-core/app/translator/platforms/base/sql/custom_types/values.py new file mode 100644 index 00000000..4edc6215 --- /dev/null +++ b/uncoder-core/app/translator/platforms/base/sql/custom_types/values.py @@ -0,0 +1,5 @@ +from app.translator.core.custom_types.values import ValueType + + +class SQLValueType(ValueType): + like_value = "like_value" diff --git a/uncoder-core/app/translator/platforms/base/sql/escape_manager.py b/uncoder-core/app/translator/platforms/base/sql/escape_manager.py index 8e5be92a..e542d898 100644 --- a/uncoder-core/app/translator/platforms/base/sql/escape_manager.py +++ b/uncoder-core/app/translator/platforms/base/sql/escape_manager.py @@ -1,14 +1,15 @@ from typing import ClassVar -from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager from app.translator.core.models.escape_details import EscapeDetails +from app.translator.platforms.base.sql.custom_types.values import SQLValueType class SQLEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { - ValueType.value: [EscapeDetails(pattern=r"(')", escape_symbols=r"'\1")], - ValueType.regex_value: [ + SQLValueType.value: [EscapeDetails(pattern=r"(')", escape_symbols=r"'\1")], + SQLValueType.like_value: [EscapeDetails(pattern=r"(['%_\\])", escape_symbols=r"\\\1")], + SQLValueType.regex_value: [ EscapeDetails(pattern=r"([$^*+()\[\]{}|.?\-\\])", escape_symbols=r"\\\1"), EscapeDetails(pattern=r"(')", escape_symbols=r"'\1"), ], diff --git a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py index 91abb7a9..e7178922 100644 --- a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py @@ -21,6 +21,7 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import LogSourceSignature from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.platforms.base.sql.custom_types.values import SQLValueType from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager @@ -56,17 +57,23 @@ def greater_or_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> st def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" - return f"{field} like '%{self._pre_process_value(field, value)}%'" + + value = f"'%{self._pre_process_value(field, value, value_type=SQLValueType.like_value)}%' escape '\\'" + return f"{field} like {value}" def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f"{field} like '%{self._pre_process_value(field, value)}'" + + value = f"'%{self._pre_process_value(field, value, value_type=SQLValueType.like_value)}' escape '\\'" + return f"{field} like {value}" def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return f"{field} like '{self._pre_process_value(field, value)}%'" + + value = f"'{self._pre_process_value(field, value, value_type=SQLValueType.like_value)}%' escape '\\'" + return f"{field} like {value}" def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): diff --git a/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py index e54e850a..3639298d 100644 --- a/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py @@ -18,7 +18,7 @@ """ import copy -from typing import ClassVar +from typing import ClassVar, Optional from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( @@ -55,20 +55,28 @@ class SQLStrValueManager(StrValueManager): "%": UnboundLenWildCard, } - def from_str_to_container(self, value: str) -> StrValue: + def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: split = [] prev_char = None for char in value: - if char in self.str_spec_symbols_map: - split.append(self.str_spec_symbols_map[char]()) - else: - if char == "'": - if prev_char == "'": - split.append(char) - prev_char = None - continue - prev_char = char + if escape_symbol and char == escape_symbol: + if prev_char == escape_symbol: + split.append(char) + prev_char = None continue + prev_char = char + continue + if not escape_symbol and char == "'": + if prev_char == "'": + split.append(char) + prev_char = None + continue + elif char in ("'", "_", "%"): + if escape_symbol and prev_char == escape_symbol: + split.append(char) + elif char in self.str_spec_symbols_map: + split.append(self.str_spec_symbols_map[char]()) + else: split.append(char) prev_char = char diff --git a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py index 2a19205c..f72714c3 100644 --- a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py @@ -29,6 +29,8 @@ from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager from app.translator.tools.utils import get_match_group +_ESCAPE_SYMBOL_GROUP_NAME = "escape_symbol" + class SqlTokenizer(QueryTokenizer): single_value_operators_map: ClassVar[dict[str, str]] = { @@ -46,9 +48,7 @@ class SqlTokenizer(QueryTokenizer): field_pattern = r'(?P"[a-zA-Z\._\-\s]+"|[a-zA-Z\._\-]+)' num_value_pattern = rf"(?P<{ValueType.number_value}>\d+(?:\.\d+)*)\s*" bool_value_pattern = rf"(?P<{ValueType.bool_value}>true|false)\s*" - single_quotes_value_pattern = ( - rf"""'(?P<{ValueType.single_quotes_value}>(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*)'""" - ) + single_quotes_value_pattern = rf"""'(?P<{ValueType.single_quotes_value}>(?:[:a-zA-Z\*0-9=+%#\-\/,_".$&^@!\(\)\{{\}}\s]|''|\\\'|\\\%|\\\_|\\\\)*)'(?:\s+escape\s+'(?P<{_ESCAPE_SYMBOL_GROUP_NAME}>.)')?""" # noqa: E501 _value_pattern = rf"{num_value_pattern}|{bool_value_pattern}|{single_quotes_value_pattern}" multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>\d+(?:,\s*\d+)*|'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*'(?:,\s*'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*')*)\)""" # noqa: E501 re_field_value_pattern = rf"""regexp_like\({field_pattern},\s*'(?P<{ValueType.regex_value}>(?:[:a-zA-Z\*\?0-9=+%#№;\-_,"\.$&^@!\{{\}}\[\]\s?<>|]|\\\'|\\)+)'\)""" # noqa: E501 @@ -71,7 +71,8 @@ def get_operator_and_value( return mapped_operator, bool_value if (s_q_value := get_match_group(match, group_name=ValueType.single_quotes_value)) is not None: - return mapped_operator, self.str_value_manager.from_str_to_container(s_q_value) + escape_symbol = get_match_group(match, group_name=_ESCAPE_SYMBOL_GROUP_NAME) + return mapped_operator, self.str_value_manager.from_str_to_container(s_q_value, escape_symbol=escape_symbol) return super().get_operator_and_value(match, mapped_operator, operator) @@ -88,10 +89,6 @@ def _search_re_field_value(self, query: str) -> Optional[tuple[FieldValue, str]] operator = Identifier(token_type=OperatorType.REGEX) return self.create_field_value(field_name, operator, value), query[match.end() :] - def tokenize(self, query: str) -> list: - query = re.sub(r"\s*ESCAPE\s*'.'", "", query) # remove `ESCAPE 'escape_char'` in LIKE expr - return super().tokenize(query) - def _get_next_token( self, query: str ) -> tuple[Union[FieldValue, FunctionValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]: diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py index e1b8708a..ffe9f71c 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -16,19 +16,21 @@ limitations under the License. ----------------------------------------------------------------- """ -from typing import ClassVar +from typing import ClassVar, Optional from app.translator.core.str_value_manager import ( BaseSpecSymbol, ReDigitalSymbol, ReWhiteSpaceSymbol, ReWordSymbol, + SingleSymbolWildCard, + StrValue, StrValueManager, ) from app.translator.platforms.elasticsearch.escape_manager import ESQLQueryEscapeManager, esql_query_escape_manager -class ESQLQueryStrValueManager(StrValueManager): +class ESQLStrValueManager(StrValueManager): escape_manager: ESQLQueryEscapeManager = esql_query_escape_manager re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { "w": ReWordSymbol, @@ -37,4 +39,13 @@ class ESQLQueryStrValueManager(StrValueManager): } -esql_query_str_value_manager = ESQLQueryStrValueManager() +class EQLStrValueManager(StrValueManager): + str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": SingleSymbolWildCard} + + def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 + split = [self.str_spec_symbols_map[char]() if char in self.str_spec_symbols_map else char for char in value] + return StrValue(value, self._concat(split)) + + +esql_str_value_manager = ESQLStrValueManager() +eql_str_value_manager = EQLStrValueManager() diff --git a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py index 751db716..65a83842 100644 --- a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ +from typing import Optional from app.translator.core.str_value_manager import ( RE_STR_SPEC_SYMBOLS_MAP, @@ -42,7 +43,7 @@ class SigmaStrValueManager(StrValueManager): } re_str_spec_symbols_map = RE_STR_SPEC_SYMBOLS_MAP - def from_str_to_container(self, value: str) -> StrValue: + def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 split = [] prev_char = None for char in value: From 449c4401e1c69b0b8d0d479893f7b49c4e1442a3 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Mon, 21 Oct 2024 11:28:13 +0300 Subject: [PATCH 6/8] fix --- .../app/translator/core/str_value_manager.py | 6 +++++- .../platforms/base/aql/str_value_manager.py | 7 ++++++- .../platforms/base/lucene/str_value_manager.py | 8 +++++++- .../platforms/base/spl/str_value_manager.py | 8 +++++++- .../platforms/base/sql/str_value_manager.py | 12 +++++++----- .../app/translator/platforms/base/sql/tokenizer.py | 7 ++++++- .../platforms/elasticsearch/str_value_manager.py | 8 +++++++- .../translator/platforms/sigma/str_value_manager.py | 8 +++++++- 8 files changed, 52 insertions(+), 12 deletions(-) diff --git a/uncoder-core/app/translator/core/str_value_manager.py b/uncoder-core/app/translator/core/str_value_manager.py index 9ba3315d..1151d7a3 100644 --- a/uncoder-core/app/translator/core/str_value_manager.py +++ b/uncoder-core/app/translator/core/str_value_manager.py @@ -182,7 +182,11 @@ class StrValueManager: container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = CONTAINER_SPEC_SYMBOLS_MAP @staticmethod - def from_str_to_container(value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG004 + def from_str_to_container( + value: str, + value_type: str = ValueType.value, # noqa: ARG004 + escape_symbol: Optional[str] = None, # noqa: ARG004 + ) -> StrValue: return StrValue(value=value, split_value=[value]) def from_re_str_to_container(self, value: str) -> StrValue: diff --git a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py index cc6ccb14..2f13931b 100644 --- a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py @@ -55,7 +55,12 @@ class AQLStrValueManager(StrValueManager): "%": UnboundLenWildCard, } - def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 + def from_str_to_container( + self, + value: str, + value_type: str = ValueType.value, # noqa: ARG002 + escape_symbol: Optional[str] = None, # noqa: ARG002 + ) -> StrValue: split = [] prev_char = None for char in value: diff --git a/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py b/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py index 4f897216..bad77b6f 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py @@ -19,6 +19,7 @@ from typing import ClassVar, Optional +from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( BaseSpecSymbol, ReAnySymbol, @@ -68,7 +69,12 @@ class LuceneStrValueManager(StrValueManager): } re_str_spec_symbols_map = RE_STR_SPEC_SYMBOLS_MAP - def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 + def from_str_to_container( + self, + value: str, + value_type: str = ValueType.value, # noqa: ARG002 + escape_symbol: Optional[str] = None, # noqa: ARG002 + ) -> StrValue: split = [] prev_char = None for char in value: diff --git a/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py b/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py index 0059e8b3..ef638d6c 100644 --- a/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py @@ -18,6 +18,7 @@ """ from typing import ClassVar, Optional +from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import BaseSpecSymbol, StrValue, StrValueManager, UnboundLenWildCard from app.translator.platforms.base.spl.escape_manager import spl_escape_manager @@ -26,7 +27,12 @@ class SplStrValueManager(StrValueManager): escape_manager = spl_escape_manager str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": UnboundLenWildCard} - def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 + def from_str_to_container( + self, + value: str, + value_type: str = ValueType.value, # noqa: ARG002 + escape_symbol: Optional[str] = None, # noqa: ARG002 + ) -> StrValue: split = [] prev_char = None for char in value: diff --git a/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py index 3639298d..5f47b8be 100644 --- a/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/sql/str_value_manager.py @@ -20,7 +20,6 @@ import copy from typing import ClassVar, Optional -from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( CONTAINER_SPEC_SYMBOLS_MAP, RE_STR_SPEC_SYMBOLS_MAP, @@ -34,6 +33,7 @@ StrValueManager, UnboundLenWildCard, ) +from app.translator.platforms.base.sql.custom_types.values import SQLValueType from app.translator.platforms.base.sql.escape_manager import sql_escape_manager SQL_CONTAINER_SPEC_SYMBOLS_MAP = copy.copy(CONTAINER_SPEC_SYMBOLS_MAP) @@ -55,7 +55,9 @@ class SQLStrValueManager(StrValueManager): "%": UnboundLenWildCard, } - def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: + def from_str_to_container( + self, value: str, value_type: str = SQLValueType.value, escape_symbol: Optional[str] = None + ) -> StrValue: split = [] prev_char = None for char in value: @@ -71,7 +73,7 @@ def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) split.append(char) prev_char = None continue - elif char in ("'", "_", "%"): + elif char in ("'", "_", "%") and value_type == SQLValueType.like_value: if escape_symbol and prev_char == escape_symbol: split.append(char) elif char in self.str_spec_symbols_map: @@ -87,13 +89,13 @@ def from_re_str_to_container(self, value: str) -> StrValue: value = value.replace("''", "'") return super().from_re_str_to_container(value) - def from_container_to_str(self, container: StrValue, value_type: str = ValueType.value) -> str: + def from_container_to_str(self, container: StrValue, value_type: str = SQLValueType.value) -> str: result = "" for el in container.split_value: if isinstance(el, str): result += self.escape_manager.escape(el, value_type) elif isinstance(el, BaseSpecSymbol): - if value_type == ValueType.regex_value: + if value_type == SQLValueType.regex_value: if isinstance(el, SingleSymbolWildCard): result += "." continue diff --git a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py index f72714c3..a68f6122 100644 --- a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py @@ -26,6 +26,7 @@ from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.tokenizer import QueryTokenizer +from app.translator.platforms.base.sql.custom_types.values import SQLValueType from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager from app.translator.tools.utils import get_match_group @@ -72,7 +73,11 @@ def get_operator_and_value( if (s_q_value := get_match_group(match, group_name=ValueType.single_quotes_value)) is not None: escape_symbol = get_match_group(match, group_name=_ESCAPE_SYMBOL_GROUP_NAME) - return mapped_operator, self.str_value_manager.from_str_to_container(s_q_value, escape_symbol=escape_symbol) + should_process_value_wildcards = self.should_process_value_wildcards(operator) + value_type = SQLValueType.like_value if should_process_value_wildcards else SQLValueType.value + return mapped_operator, self.str_value_manager.from_str_to_container( + s_q_value, value_type=value_type, escape_symbol=escape_symbol + ) return super().get_operator_and_value(match, mapped_operator, operator) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py index ffe9f71c..1de55c91 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -18,6 +18,7 @@ """ from typing import ClassVar, Optional +from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( BaseSpecSymbol, ReDigitalSymbol, @@ -42,7 +43,12 @@ class ESQLStrValueManager(StrValueManager): class EQLStrValueManager(StrValueManager): str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": SingleSymbolWildCard} - def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 + def from_str_to_container( + self, + value: str, + value_type: str = ValueType.value, # noqa: ARG002 + escape_symbol: Optional[str] = None, # noqa: ARG002 + ) -> StrValue: split = [self.str_spec_symbols_map[char]() if char in self.str_spec_symbols_map else char for char in value] return StrValue(value, self._concat(split)) diff --git a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py index 65a83842..6d3abe56 100644 --- a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py @@ -18,6 +18,7 @@ """ from typing import Optional +from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( RE_STR_SPEC_SYMBOLS_MAP, ReDigitalSymbol, @@ -43,7 +44,12 @@ class SigmaStrValueManager(StrValueManager): } re_str_spec_symbols_map = RE_STR_SPEC_SYMBOLS_MAP - def from_str_to_container(self, value: str, escape_symbol: Optional[str] = None) -> StrValue: # noqa: ARG002 + def from_str_to_container( + self, + value: str, + value_type: str = ValueType.value, # noqa: ARG002 + escape_symbol: Optional[str] = None # noqa: ARG002 + ) -> StrValue: split = [] prev_char = None for char in value: From 691a3b3a097b1a2648cef323931ebe2d3e600d29 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Mon, 21 Oct 2024 15:32:04 +0300 Subject: [PATCH 7/8] fix --- uncoder-core/app/translator/platforms/base/sql/tokenizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py index a68f6122..fe92c8f6 100644 --- a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py @@ -49,7 +49,7 @@ class SqlTokenizer(QueryTokenizer): field_pattern = r'(?P"[a-zA-Z\._\-\s]+"|[a-zA-Z\._\-]+)' num_value_pattern = rf"(?P<{ValueType.number_value}>\d+(?:\.\d+)*)\s*" bool_value_pattern = rf"(?P<{ValueType.bool_value}>true|false)\s*" - single_quotes_value_pattern = rf"""'(?P<{ValueType.single_quotes_value}>(?:[:a-zA-Z\*0-9=+%#\-\/,_".$&^@!\(\)\{{\}}\s]|''|\\\'|\\\%|\\\_|\\\\)*)'(?:\s+escape\s+'(?P<{_ESCAPE_SYMBOL_GROUP_NAME}>.)')?""" # noqa: E501 + single_quotes_value_pattern = rf"""'(?P<{ValueType.single_quotes_value}>(?:[:a-zA-Z\*0-9=+%#\-\/,_".$&^@!\(\)\{{\}}\s]|''|\\\'|\\\%|\\\_|\\\\|\\)*)'(?:\s+escape\s+'(?P<{_ESCAPE_SYMBOL_GROUP_NAME}>.)')?""" # noqa: E501 _value_pattern = rf"{num_value_pattern}|{bool_value_pattern}|{single_quotes_value_pattern}" multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>\d+(?:,\s*\d+)*|'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*'(?:,\s*'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*')*)\)""" # noqa: E501 re_field_value_pattern = rf"""regexp_like\({field_pattern},\s*'(?P<{ValueType.regex_value}>(?:[:a-zA-Z\*\?0-9=+%#№;\-_,"\.$&^@!\{{\}}\[\]\s?<>|]|\\\'|\\)+)'\)""" # noqa: E501 From a42682827dbfe25ace69355c4b71ccab2318591b Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Tue, 22 Oct 2024 08:57:23 +0300 Subject: [PATCH 8/8] fix anomali --- .../platforms/anomali/renders/anomali.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/uncoder-core/app/translator/platforms/anomali/renders/anomali.py b/uncoder-core/app/translator/platforms/anomali/renders/anomali.py index 531cc739..6f9e89f6 100644 --- a/uncoder-core/app/translator/platforms/anomali/renders/anomali.py +++ b/uncoder-core/app/translator/platforms/anomali/renders/anomali.py @@ -28,6 +28,27 @@ class AnomaliFieldValueRender(SqlFieldValueRender): details: PlatformDetails = anomali_query_details + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" + + value = f"'%{self._pre_process_value(field, value)}%'" + return f"{field} like {value}" + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" + + value = f"'%{self._pre_process_value(field, value)}'" + return f"{field} like {value}" + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" + + value = f"'{self._pre_process_value(field, value)}%'" + return f"{field} like {value}" + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f'message contains "{self._pre_process_value(field, value)}"' pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy