From 2e398a8b473cff3a6300999928a5d5aa652a2e27 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Tue, 10 Sep 2024 15:40:16 +0300 Subject: [PATCH 1/6] gis-8639 add elastic-eql-query parser --- .../platforms/elasticsearch/__init__.py | 8 +- .../platforms/elasticsearch/const.py | 73 +++++++++++++++++++ .../parsers/elasticsearch_eql.py | 37 ++++++++++ .../elasticsearch/str_value_manager.py | 51 +++++++++++++ .../platforms/elasticsearch/tokenizer.py | 66 +++++++++++++++++ 5 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py diff --git a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py index 96017e2e..a9c84be4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py @@ -1,8 +1,14 @@ -from app.translator.platforms.elasticsearch.parsers.detection_rule import ElasticSearchRuleParser # noqa: F401 +from app.translator.platforms.elasticsearch.parsers.detection_rule import ( + ElasticSearchRuleParser, # noqa: F401 + ElasticSearchRuleTOMLParser, # noqa: F401 +) from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser # noqa: F401 +from app.translator.platforms.elasticsearch.parsers.elasticsearch_eql import ElasticSearchEQLQueryParser # noqa: F401 from app.translator.platforms.elasticsearch.renders.detection_rule import ElasticSearchRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elasticsearch_cti import ElasticsearchCTI # noqa: F401 +from app.translator.platforms.elasticsearch.renders.esql import ESQLQueryRender # noqa: F401 +from app.translator.platforms.elasticsearch.renders.esql_rule import ESQLRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.kibana import KibanaRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.xpack_watcher import XPackWatcherRuleRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/elasticsearch/const.py b/uncoder-core/app/translator/platforms/elasticsearch/const.py index 08409610..59a50ac3 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/const.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/const.py @@ -5,9 +5,13 @@ _ELASTIC_LUCENE_QUERY = "elastic-lucene-query" _ELASTIC_LUCENE_RULE = "elastic-lucene-rule" +_ELASTIC_LUCENE_RULE_TOML = "elastic-lucene-rule-toml" _ELASTIC_KIBANA_RULE = "elastic-kibana-rule" _ELASTALERT_LUCENE_RULE = "elastalert-lucene-rule" _ELASTIC_WATCHER_RULE = "elastic-watcher-rule" +_ELASTIC_ESQL_QUERY = "elastic-esql-query" +_ELASTIC_ESQL_RULE = "elastic-esql-rule" +_ELASTIC_EQL_QUERY = "elastic-eql-query" ELASTIC_QUERY_TYPES = { _ELASTIC_LUCENE_QUERY, @@ -15,6 +19,8 @@ _ELASTIC_KIBANA_RULE, _ELASTALERT_LUCENE_RULE, _ELASTIC_WATCHER_RULE, + _ELASTIC_ESQL_QUERY, + _ELASTIC_ESQL_RULE, } ELASTICSEARCH_LUCENE_QUERY_DETAILS = { @@ -24,6 +30,20 @@ **PLATFORM_DETAILS, } +ELASTICSEARCH_ESQL_QUERY_DETAILS = { + "platform_id": _ELASTIC_ESQL_QUERY, + "name": "Elasticsearch ES|QL Query", + "platform_name": "Query (ES|QL)", + **PLATFORM_DETAILS, +} + +ELASTICSEARCH_ESQL_RULE_DETAILS = { + "platform_id": _ELASTIC_ESQL_RULE, + "name": "Elasticsearch ES|QL Rule", + "platform_name": "Rule (ES|QL)", + **PLATFORM_DETAILS, +} + ELASTICSEARCH_RULE_DETAILS = { "platform_id": _ELASTIC_LUCENE_RULE, "name": "Elastic Rule", @@ -32,6 +52,14 @@ **PLATFORM_DETAILS, } +ELASTICSEARCH_RULE_TOML_DETAILS = { + "platform_id": _ELASTIC_LUCENE_RULE_TOML, + "name": "Elastic Rule TOML", + "platform_name": "Detection Rule (Lucene) TOML", + "first_choice": 0, + **PLATFORM_DETAILS, +} + KIBANA_DETAILS = { "platform_id": _ELASTIC_KIBANA_RULE, "name": "Elastic Kibana Saved Search", @@ -56,11 +84,22 @@ **PLATFORM_DETAILS, } +ELASTICSEARCH_EQL_QUERY_DETAILS = { + "platform_id": _ELASTIC_EQL_QUERY, + "name": "Elasticsearch EQL Query", + "platform_name": "Query (EQL)", + **PLATFORM_DETAILS, +} + elasticsearch_lucene_query_details = PlatformDetails(**ELASTICSEARCH_LUCENE_QUERY_DETAILS) +elasticsearch_esql_query_details = PlatformDetails(**ELASTICSEARCH_ESQL_QUERY_DETAILS) +elasticsearch_esql_rule_details = PlatformDetails(**ELASTICSEARCH_ESQL_RULE_DETAILS) elasticsearch_rule_details = PlatformDetails(**ELASTICSEARCH_RULE_DETAILS) +elasticsearch_rule_toml_details = PlatformDetails(**ELASTICSEARCH_RULE_TOML_DETAILS) elastalert_details = PlatformDetails(**ELASTALERT_DETAILS) kibana_rule_details = PlatformDetails(**KIBANA_DETAILS) xpack_watcher_details = PlatformDetails(**XPACK_WATCHER_DETAILS) +elastic_eql_query_details = PlatformDetails(**ELASTICSEARCH_EQL_QUERY_DETAILS) ELASTICSEARCH_DETECTION_RULE = { "description": "Autogenerated ElasticSearch Detection Rule.", @@ -167,3 +206,37 @@ } }, } + +ESQL_RULE = { + "name": "", + "tags": [], + "interval": "5m", + "enabled": True, + "revision": 0, + "description": "", + "risk_score": 21, + "severity": "low", + "license": "", + "output_index": "", + "meta": {"from": "1m"}, + "author": [], + "false_positives": [], + "from": "now-360s", + "rule_id": "", + "max_signals": 100, + "risk_score_mapping": [], + "severity_mapping": [], + "threat": [], + "to": "now", + "references": [], + "version": 1, + "exceptions_list": [], + "immutable": False, + "related_integrations": [], + "required_fields": [], + "setup": "", + "type": "esql", + "language": "esql", + "query": "", + "actions": [], +} diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py new file mode 100644 index 00000000..7a4d42b3 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py @@ -0,0 +1,37 @@ +import re + +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser +from app.translator.managers import parser_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings +from app.translator.platforms.elasticsearch.const import elastic_eql_query_details +from app.translator.platforms.elasticsearch.mapping import elasticsearch_lucene_query_mappings +from app.translator.platforms.elasticsearch.tokenizer import ElasticSearchEQLTokenizer + + +@parser_manager.register_supported_by_roota +class ElasticSearchEQLQueryParser(PlatformQueryParser): + details: PlatformDetails = elastic_eql_query_details + tokenizer = ElasticSearchEQLTokenizer() + mappings: LuceneMappings = elasticsearch_lucene_query_mappings + query_delimiter_pattern = r"\swhere\s" + + def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: + log_source = {"category": []} + if re.search(self.query_delimiter_pattern, query, flags=re.IGNORECASE): + sp_query = re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE) + if sp_query[0].lower() != "all": + log_source["category"].append(sp_query[0]) + return sp_query[1], log_source + return query, log_source + + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + query, log_sources = self._parse_query(raw_query_container.query) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(field_tokens, log_sources) + meta_info = raw_query_container.meta_info + meta_info.query_fields = field_tokens + meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py new file mode 100644 index 00000000..48b153a2 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -0,0 +1,51 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from typing import ClassVar + +from app.translator.core.str_value_manager import ( + BaseSpecSymbol, + ReDigitalSymbol, + ReWhiteSpaceSymbol, + ReWordSymbol, + SingleSymbolWildCard, + StrValue, + StrValueManager, +) +from app.translator.platforms.elasticsearch.escape_manager import ESQLQueryEscapeManager, esql_query_escape_manager + + +class ESQLQueryStrValueManager(StrValueManager): + escape_manager: ESQLQueryEscapeManager = esql_query_escape_manager + re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { + "w": ReWordSymbol, + "d": ReDigitalSymbol, + "s": ReWhiteSpaceSymbol, + } + + +class ElasticEQLQueryStrValueManager(StrValueManager): + str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": SingleSymbolWildCard} + + def from_str_to_container(self, value: str) -> StrValue: + split = [self.str_spec_symbols_map[char]() if char in self.str_spec_symbols_map else char for char in value] + return StrValue(value, self._concat(split)) + + +esql_query_str_value_manager = ESQLQueryStrValueManager() +elastic_eql_str_value_manager = ElasticEQLQueryStrValueManager() diff --git a/uncoder-core/app/translator/platforms/elasticsearch/tokenizer.py b/uncoder-core/app/translator/platforms/elasticsearch/tokenizer.py index 9f6136d2..1bc3bf44 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/tokenizer.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/tokenizer.py @@ -15,9 +15,75 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ +import re +from typing import Any, ClassVar, Optional, Union +from app.translator.core.custom_types.tokens import OperatorType +from app.translator.core.custom_types.values import ValueType +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.base.lucene.tokenizer import LuceneTokenizer +from app.translator.platforms.elasticsearch.str_value_manager import elastic_eql_str_value_manager +from app.translator.tools.utils import get_match_group class ElasticSearchTokenizer(LuceneTokenizer): pass + + +class ElasticSearchEQLTokenizer(QueryTokenizer): + single_value_operators_map: ClassVar[dict[str, str]] = { + ":": OperatorType.EQ, + "==": OperatorType.EQ, + "<=": OperatorType.LTE, + "<": OperatorType.LT, + ">=": OperatorType.GTE, + ">": OperatorType.GT, + "!=": OperatorType.NOT_EQ, + "regex~": OperatorType.REGEX, + "regex": OperatorType.REGEX, + } + + multi_value_operators_map: ClassVar[dict[str, str]] = { + "in": OperatorType.EQ, + "in~": OperatorType.EQ, + ":": OperatorType.EQ, + } + wildcard_symbol = "*" + field_pattern = r"(?P[a-zA-Z\.\-_`]+)" + re_value_pattern = ( + rf'"(?P<{ValueType.regex_value}>(?:[:a-zA-Z*0-9=+%#\-_/,;`?~‘\'.<>$&^@!\]\[()\s]|\\\"|\\)*)\[\^[z|Z]\]\.\?"' # noqa: RUF001 + ) + double_quotes_value_pattern = ( + rf'"(?P<{ValueType.double_quotes_value}>(?:[:a-zA-Z*0-9=+%#\-_/,;`?~‘\'.<>$&^@!\]\[()\s]|\\\"|\\)*)"' # noqa: RUF001 + ) + _value_pattern = rf"{re_value_pattern}|{double_quotes_value_pattern}" + multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>[:a-zA-Z\"\*0-9=+%#№;\-_\/\\'\,.$&^@!\(\[\]\s|]+)\)""" + multi_value_check_pattern = r"___field___\s*___operator___\s*\(" + keyword_pattern = ( + rf'"(?P<{ValueType.double_quotes_value}>(?:[:a-zA-Z*0-9=+%#\-_/,;`?~‘\'.<>$&^@!\]\[()\s]|\\\"|\\)*)"' # noqa: RUF001 + ) + + str_value_manager = elastic_eql_str_value_manager + + def get_operator_and_value( + self, match: re.Match, mapped_operator: str = OperatorType.EQ, operator: Optional[str] = None + ) -> tuple[str, Any]: + if (re_value := get_match_group(match, group_name=ValueType.regex_value)) is not None: + return OperatorType.REGEX, self.str_value_manager.from_re_str_to_container(re_value) + + if (d_q_value := get_match_group(match, group_name=ValueType.double_quotes_value)) is not None: + return mapped_operator, self.str_value_manager.from_str_to_container(d_q_value) + + return super().get_operator_and_value(match, mapped_operator, operator) + + def is_multi_value_flow(self, field_name: str, operator: str, query: str) -> bool: + check_pattern = self.multi_value_check_pattern + check_regex = check_pattern.replace("___field___", field_name).replace("___operator___", operator) + return bool(re.match(check_regex, query)) + + @staticmethod + def create_field_value(field_name: str, operator: Identifier, value: Union[str, list]) -> FieldValue: + field_name = field_name.replace("`", "") + return FieldValue(source_name=field_name, operator=operator, value=value) From 83958ed3092da1e76c1d93f2a3405ef2c39a3a90 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Tue, 10 Sep 2024 15:49:32 +0300 Subject: [PATCH 2/6] gis-8639 fix mapping --- .../translator/platforms/elasticsearch/mapping.py | 15 +++++++++++++++ .../elasticsearch/parsers/elasticsearch_eql.py | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py index b0489fbf..2ee6cae3 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py @@ -1,12 +1,16 @@ from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import ( elastalert_details, + elastic_eql_query_details, + elasticsearch_esql_query_details, elasticsearch_lucene_query_details, elasticsearch_rule_details, kibana_rule_details, xpack_watcher_details, ) +DEFAULT_MAPPING_NAME = "default" + elasticsearch_lucene_query_mappings = LuceneMappings( platform_dir="elasticsearch", platform_details=elasticsearch_lucene_query_details ) @@ -14,3 +18,14 @@ elastalert_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=elastalert_details) kibana_rule_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=kibana_rule_details) xpack_watcher_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=xpack_watcher_details) +elastic_eql_query_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=elastic_eql_query_details) + + +class ElasticESQLMappings(LuceneMappings): + is_strict_mapping: bool = True + skip_load_default_mappings = True + + +esql_query_mappings = ElasticESQLMappings( + platform_dir="elasticsearch_esql", platform_details=elasticsearch_esql_query_details +) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py index 7a4d42b3..9ee7e0d4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py @@ -6,7 +6,7 @@ from app.translator.managers import parser_manager from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import elastic_eql_query_details -from app.translator.platforms.elasticsearch.mapping import elasticsearch_lucene_query_mappings +from app.translator.platforms.elasticsearch.mapping import elastic_eql_query_mappings from app.translator.platforms.elasticsearch.tokenizer import ElasticSearchEQLTokenizer @@ -14,7 +14,7 @@ class ElasticSearchEQLQueryParser(PlatformQueryParser): details: PlatformDetails = elastic_eql_query_details tokenizer = ElasticSearchEQLTokenizer() - mappings: LuceneMappings = elasticsearch_lucene_query_mappings + mappings: LuceneMappings = elastic_eql_query_mappings query_delimiter_pattern = r"\swhere\s" def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: From 90f21791d5787975cfa99e1d6be8c56613fa86c2 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:04:41 +0300 Subject: [PATCH 3/6] gis-8639 fix --- .../translator/platforms/elasticsearch/str_value_manager.py | 4 ++-- .../app/translator/platforms/elasticsearch/tokenizer.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py index 48b153a2..1c0e959b 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -39,7 +39,7 @@ class ESQLQueryStrValueManager(StrValueManager): } -class ElasticEQLQueryStrValueManager(StrValueManager): +class EQLQueryStrValueManager(StrValueManager): str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": SingleSymbolWildCard} def from_str_to_container(self, value: str) -> StrValue: @@ -48,4 +48,4 @@ def from_str_to_container(self, value: str) -> StrValue: esql_query_str_value_manager = ESQLQueryStrValueManager() -elastic_eql_str_value_manager = ElasticEQLQueryStrValueManager() +eql_str_value_manager = EQLQueryStrValueManager() diff --git a/uncoder-core/app/translator/platforms/elasticsearch/tokenizer.py b/uncoder-core/app/translator/platforms/elasticsearch/tokenizer.py index 1bc3bf44..115144e8 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/tokenizer.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/tokenizer.py @@ -24,7 +24,7 @@ from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.base.lucene.tokenizer import LuceneTokenizer -from app.translator.platforms.elasticsearch.str_value_manager import elastic_eql_str_value_manager +from app.translator.platforms.elasticsearch.str_value_manager import eql_str_value_manager from app.translator.tools.utils import get_match_group @@ -65,7 +65,7 @@ class ElasticSearchEQLTokenizer(QueryTokenizer): rf'"(?P<{ValueType.double_quotes_value}>(?:[:a-zA-Z*0-9=+%#\-_/,;`?~‘\'.<>$&^@!\]\[()\s]|\\\"|\\)*)"' # noqa: RUF001 ) - str_value_manager = elastic_eql_str_value_manager + str_value_manager = eql_str_value_manager def get_operator_and_value( self, match: re.Match, mapped_operator: str = OperatorType.EQ, operator: Optional[str] = None From 1c1333fc31dd2600913507c8539e8dddb8a67abe Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:12:52 +0300 Subject: [PATCH 4/6] gis-8639 fix --- .../platforms/elasticsearch/renders/esql.py | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py new file mode 100644 index 00000000..c15e7f45 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -0,0 +1,139 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from typing import Optional, Union + +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.custom_types.values import ValueType +from app.translator.core.exceptions.render import UnsupportedRenderMethod +from app.translator.core.mapping import LogSourceSignature +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.managers import render_manager +from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details +from app.translator.platforms.elasticsearch.mapping import ElasticESQLMappings, esql_query_mappings +from app.translator.platforms.elasticsearch.str_value_manager import ( + ESQLStrValueManager, + esql_str_value_manager, +) + + +class ESQLFieldValueRender(BaseFieldValueRender): + details: PlatformDetails = elasticsearch_esql_query_details + str_value_manager: ESQLStrValueManager = esql_str_value_manager + + @staticmethod + def _make_case_insensitive(value: str) -> str: + container: list[str] = [] + for v in value: + if v.isalpha(): + container.append(f"[{v.upper()}{v.lower()}]") + else: + container.append(v) + return "".join(container) + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f'"{value}"' + + @staticmethod + def _wrap_int_value(value: int) -> str: + return f'"{value}"' + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} == {value}" + + def less_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} < {value}" + + def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} <= {value}" + + def greater_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} > {value}" + + def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} >= {value}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} != {value}" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" + if field.endswith(".text"): + return self.regex_modifier(field=field, value=value) + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False, wrap_int=True) + return f'{field} like "*{value}*"' + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field.endswith(".text"): + return self.regex_modifier(field=field, value=value) + if isinstance(value, list): + return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"ends_with({field}, {value})" + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field.endswith(".text"): + return self.regex_modifier(field=field, value=value) + if isinstance(value, list): + return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"starts_with({field}, {value})" + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=False, wrap_int=True) + if isinstance(value, str): + value = self._make_case_insensitive(value) + return f'{field} rlike ".*{value}.*"' + + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 + raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords") + + +@render_manager.register +class ESQLQueryRender(PlatformQueryRender): + details: PlatformDetails = elasticsearch_esql_query_details + mappings: ElasticESQLMappings = esql_query_mappings + comment_symbol = "//" + + or_token = "or" + and_token = "and" + not_token = "not" + field_value_render = ESQLFieldValueRender(or_token=or_token) + + def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002 + table = str(log_source_signature) if str(log_source_signature) else "*" + return f"FROM {table} |" + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"WHERE {query}" if query else "" From d6bd1f9ab5477ffe6b7719374ff4ad8f08db528e Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:13:01 +0300 Subject: [PATCH 5/6] gis-8639 fix --- .../platforms/elasticsearch/str_value_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py index 1c0e959b..88be99fa 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -30,7 +30,7 @@ from app.translator.platforms.elasticsearch.escape_manager import ESQLQueryEscapeManager, esql_query_escape_manager -class ESQLQueryStrValueManager(StrValueManager): +class ESQLStrValueManager(StrValueManager): escape_manager: ESQLQueryEscapeManager = esql_query_escape_manager re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { "w": ReWordSymbol, @@ -39,7 +39,7 @@ class ESQLQueryStrValueManager(StrValueManager): } -class EQLQueryStrValueManager(StrValueManager): +class EQLStrValueManager(StrValueManager): str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": SingleSymbolWildCard} def from_str_to_container(self, value: str) -> StrValue: @@ -47,5 +47,5 @@ def from_str_to_container(self, value: str) -> StrValue: return StrValue(value, self._concat(split)) -esql_query_str_value_manager = ESQLQueryStrValueManager() -eql_str_value_manager = EQLQueryStrValueManager() +esql_str_value_manager = ESQLStrValueManager() +eql_str_value_manager = EQLStrValueManager() From 608a1f4de2f901fa388c5f7135438a928e0a7cc1 Mon Sep 17 00:00:00 2001 From: Nazar Gesyk Date: Wed, 16 Oct 2024 10:40:38 +0300 Subject: [PATCH 6/6] gis-8639 fixes --- .../app/translator/platforms/elasticsearch/renders/esql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 9e71fe2a..62e785ee 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -29,13 +29,13 @@ from app.translator.platforms.elasticsearch.mapping import ElasticESQLMappings, esql_query_mappings from app.translator.platforms.elasticsearch.str_value_manager import ( ESQLQueryStrValueManager, - esql_query_str_value_manager + esql_str_value_manager ) class ESQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = elasticsearch_esql_query_details - str_value_manager: ESQLQueryStrValueManager = esql_query_str_value_manager + str_value_manager: ESQLQueryStrValueManager = esql_str_value_manager @staticmethod def _make_case_insensitive(value: str) -> str: pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy