From e935a3dd5fc960273203f1262de2d4fe03913197 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 11 Jun 2024 09:25:27 +0200 Subject: [PATCH 01/40] esql rule render --- .../platforms/elasticsearch/__init__.py | 2 + .../platforms/elasticsearch/const.py | 59 ++++++++++ .../platforms/elasticsearch/renders/esql.py | 62 ++++++++++ .../elasticsearch/renders/esql_rule.py | 110 ++++++++++++++++++ 4 files changed, 233 insertions(+) create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py diff --git a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py index 96017e2e..3c53e3cf 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py @@ -6,3 +6,5 @@ from app.translator.platforms.elasticsearch.renders.elasticsearch_cti import ElasticsearchCTI # noqa: F401 from app.translator.platforms.elasticsearch.renders.kibana import KibanaRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.xpack_watcher import XPackWatcherRuleRender # noqa: F401 +from app.translator.platforms.elasticsearch.renders.esql import ESQLQueryRender # noqa: F401 +from app.translator.platforms.elasticsearch.renders.esql_rule import ESQLRuleRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/elasticsearch/const.py b/uncoder-core/app/translator/platforms/elasticsearch/const.py index 08409610..b22ffd33 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/const.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/const.py @@ -8,6 +8,8 @@ _ELASTIC_KIBANA_RULE = "elastic-kibana-rule" _ELASTALERT_LUCENE_RULE = "elastalert-lucene-rule" _ELASTIC_WATCHER_RULE = "elastic-watcher-rule" +_ELASTIC_ESQL_QUERY = "elastic-esql-query" +_ELASTIC_ESQL_RULE = "elastic-esql-rule" ELASTIC_QUERY_TYPES = { _ELASTIC_LUCENE_QUERY, @@ -15,6 +17,8 @@ _ELASTIC_KIBANA_RULE, _ELASTALERT_LUCENE_RULE, _ELASTIC_WATCHER_RULE, + _ELASTIC_ESQL_QUERY, + _ELASTIC_ESQL_RULE } ELASTICSEARCH_LUCENE_QUERY_DETAILS = { @@ -24,6 +28,20 @@ **PLATFORM_DETAILS, } +ELASTICSEARCH_ESQL_QUERY_DETAILS = { + "platform_id": _ELASTIC_ESQL_QUERY, + "name": "Elasticsearch ES|QL Query", + "platform_name": "Query (ES|QL)", + **PLATFORM_DETAILS, +} + +ELASTICSEARCH_ESQL_RULE_DETAILS = { + "platform_id": _ELASTIC_ESQL_RULE, + "name": "Elasticsearch ES|QL Rule", + "platform_name": "Rule (ES|QL)", + **PLATFORM_DETAILS, +} + ELASTICSEARCH_RULE_DETAILS = { "platform_id": _ELASTIC_LUCENE_RULE, "name": "Elastic Rule", @@ -57,6 +75,8 @@ } elasticsearch_lucene_query_details = PlatformDetails(**ELASTICSEARCH_LUCENE_QUERY_DETAILS) +elasticsearch_esql_query_details = PlatformDetails(**ELASTICSEARCH_ESQL_QUERY_DETAILS) +elasticsearch_esql_rule_details = PlatformDetails(**ELASTICSEARCH_ESQL_RULE_DETAILS) elasticsearch_rule_details = PlatformDetails(**ELASTICSEARCH_RULE_DETAILS) elastalert_details = PlatformDetails(**ELASTALERT_DETAILS) kibana_rule_details = PlatformDetails(**KIBANA_DETAILS) @@ -167,3 +187,42 @@ } }, } + +ESQL_RULE = { + "id": "49f252d0-8b18-4c70-8440-919f6b20c258", + "updated_at": "2024-06-10T10:24:30.349Z", + "updated_by": "elastic", + "created_at": "2024-06-10T10:24:30.349Z", + "created_by": "elastic", + "name": "", + "tags": [], + "interval": "5m", + "enabled": True, + "revision": 0, + "description": "", + "risk_score": 21, + "severity": "low", + "license": "", + "output_index": "", + "meta": {"from": "1m"}, + "author": [], + "false_positives": [], + "from": "now-360s", + "rule_id": "", + "max_signals": 100, + "risk_score_mapping": [], + "severity_mapping": [], + "threat": [], + "to": "now", + "references": [], + "version": 1, + "exceptions_list": [], + "immutable": False, + "related_integrations": [], + "required_fields": [], + "setup": "", + "type": "esql", + "language": "esql", + "query": "", + "actions": [] +} \ No newline at end of file diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py new file mode 100644 index 00000000..4c2efa77 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -0,0 +1,62 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.mapping import LogSourceSignature +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.managers import render_manager +from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details + + +class ESQLFieldValue(SqlFieldValue): + details: PlatformDetails = elasticsearch_esql_query_details + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" + return f'{field} LIKE "*{value}*"' + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" + return f'{field} LIKE "*{value}?"' + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" + return f'{field} LIKE "{value}%"' + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" + return f'{field} RLIKE "{value}"' + + +@render_manager.register +class ESQLQueryRender(SqlQueryRender): + details: PlatformDetails = elasticsearch_esql_query_details + mappings: ElasticSearchMappings = elasticsearch_mappings + + or_token = "OR" + field_value_map = ESQLFieldValue(or_token=or_token) + + def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 + table = str(log_source_signature) if str(log_source_signature) else "*" + return f"FROM {table} metadata _id, _version, _index |" \ No newline at end of file diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py new file mode 100644 index 00000000..c9f20d96 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -0,0 +1,110 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +import copy +from datetime import datetime +import json +from typing import Optional, Union +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.mapping import LogSourceSignature, SourceMapping +from app.translator.core.mitre import MitreConfig +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer +from app.translator.managers import render_manager +from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.const import elasticsearch_esql_rule_details, ESQL_RULE +from app.translator.platforms.elasticsearch.renders.esql import ESQLFieldValue, ESQLQueryRender + + +_AUTOGENERATED_TEMPLATE = "Autogenerated ESQL Rule" + + +class ESQLRuleFieldValue(ESQLFieldValue): + details: PlatformDetails = elasticsearch_esql_rule_details + + + +@render_manager.register +class ESQLRuleRender(ESQLQueryRender): + details: PlatformDetails = elasticsearch_esql_rule_details + mappings: ElasticSearchMappings = elasticsearch_mappings + mitre: MitreConfig = MitreConfig() + + + def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: + if not mitre_attack.get("techniques"): + return [] + threat = [] + + for tactic in mitre_attack["tactics"]: + tactic_render = {"id": tactic["external_id"], "name": tactic["tactic"], "reference": tactic["url"]} + sub_threat = {"tactic": tactic_render, "framework": "MITRE ATT&CK", "technique": []} + for technique in mitre_attack["techniques"]: + technique_id = technique["technique_id"].lower() + if "." in technique_id: + technique_id = technique_id[: technique["technique_id"].index(".")] + main_technique = self.mitre.get_technique(technique_id) + if tactic["tactic"] in main_technique["tactic"]: + sub_threat["technique"].append( + { + "id": main_technique["technique_id"], + "name": main_technique["technique"], + "reference": main_technique["url"], + } + ) + if len(sub_threat["technique"]) > 0: + threat.append(sub_threat) + + return sorted(threat, key=lambda x: x["tactic"]["id"]) + + def finalize_query( + self, + prefix: str, + query: str, + functions: str, + meta_info: MetaInfoContainer, + source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 + not_supported_functions: Optional[list] = None, + *args, # noqa: ARG002 + **kwargs, # noqa: ARG002 + ) -> str: + query = super().finalize_query(prefix=prefix, query=query, functions=functions) + rule = copy.deepcopy(ESQL_RULE) + timestamp = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + rule.update( + { + "updated_at": timestamp, + "created_at": timestamp, + "query": query, + "description": meta_info.description or rule["description"] or _AUTOGENERATED_TEMPLATE, + "name": meta_info.title or _AUTOGENERATED_TEMPLATE, + "rule_id": meta_info.id, + "author": [meta_info.author], + "severity": meta_info.severity, + "references": meta_info.references, + "license": meta_info.license, + "tags": meta_info.tags, + "threat": self.__create_mitre_threat(meta_info.mitre_attack), + "false_positives": meta_info.false_positives, + } + ) + rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) + if not_supported_functions: + rendered_not_supported = self.render_not_supported_functions(not_supported_functions) + return rule_str + rendered_not_supported + return rule_str \ No newline at end of file From e0188ccdcfddc35067ca76a8f71759b1e78cee95 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:14:45 +0200 Subject: [PATCH 02/40] remove redundant fields --- .../app/translator/platforms/elasticsearch/const.py | 5 ----- .../platforms/elasticsearch/renders/esql_rule.py | 7 +------ 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/const.py b/uncoder-core/app/translator/platforms/elasticsearch/const.py index b22ffd33..9a93c31e 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/const.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/const.py @@ -189,11 +189,6 @@ } ESQL_RULE = { - "id": "49f252d0-8b18-4c70-8440-919f6b20c258", - "updated_at": "2024-06-10T10:24:30.349Z", - "updated_by": "elastic", - "created_at": "2024-06-10T10:24:30.349Z", - "created_by": "elastic", "name": "", "tags": [], "interval": "5m", diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py index c9f20d96..21ac4f36 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -17,11 +17,9 @@ ----------------------------------------------------------------- """ import copy -from datetime import datetime import json from typing import Optional, Union -from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.mapping import LogSourceSignature, SourceMapping +from app.translator.core.mapping import SourceMapping from app.translator.core.mitre import MitreConfig from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer @@ -85,11 +83,8 @@ def finalize_query( ) -> str: query = super().finalize_query(prefix=prefix, query=query, functions=functions) rule = copy.deepcopy(ESQL_RULE) - timestamp = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') rule.update( { - "updated_at": timestamp, - "created_at": timestamp, "query": query, "description": meta_info.description or rule["description"] or _AUTOGENERATED_TEMPLATE, "name": meta_info.title or _AUTOGENERATED_TEMPLATE, From b74fb3fa0cdacc2f0a61dfeafe1e3654b861a4f3 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 18 Jun 2024 09:20:45 +0200 Subject: [PATCH 03/40] escape manager and strvalue manager --- .../platforms/elasticsearch/escape_manager.py | 18 +++++ .../platforms/elasticsearch/renders/esql.py | 6 +- .../elasticsearch/str_value_manager.py | 80 +++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py diff --git a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py new file mode 100644 index 00000000..8fd4d474 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py @@ -0,0 +1,18 @@ +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.escape_manager import EscapeManager +from app.translator.core.models.escape_details import EscapeDetails + + +class ESQLEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern=r'"', escape_symbols=r'\\"')], + ValueType.regex_value: [ + EscapeDetails(pattern=r"([$^*+()\[\]{}|.?\-\\])", escape_symbols=r"\\\1"), + EscapeDetails(pattern=r"(')", escape_symbols=r"'\1"), + ], + } + + +esql_escape_manager = ESQLEscapeManager() diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 4c2efa77..ca89328d 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -23,10 +23,12 @@ from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details +from app.translator.platforms.elasticsearch.str_value_manager import esql_str_value_manager class ESQLFieldValue(SqlFieldValue): details: PlatformDetails = elasticsearch_esql_query_details + str_value_manager = esql_str_value_manager def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): @@ -54,7 +56,9 @@ class ESQLQueryRender(SqlQueryRender): details: PlatformDetails = elasticsearch_esql_query_details mappings: ElasticSearchMappings = elasticsearch_mappings - or_token = "OR" + or_token = "or" + and_token = "and" + not_token = "not" field_value_map = ESQLFieldValue(or_token=or_token) def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py new file mode 100644 index 00000000..e7727ba4 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -0,0 +1,80 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +import copy +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.str_value_manager import ( + CONTAINER_SPEC_SYMBOLS_MAP, + BaseSpecSymbol, + ReAnySymbol, + ReCaretSymbol, + ReCommaSymbol, + ReDigitalSymbol, + ReEndOfStrSymbol, + ReHyphenSymbol, + ReLeftCurlyBracket, + ReLeftParenthesis, + ReLeftSquareBracket, + ReOneOrMoreQuantifier, + ReOrOperator, + ReRightCurlyBracket, + ReRightParenthesis, + ReRightSquareBracket, + ReWhiteSpaceSymbol, + ReWordSymbol, + ReZeroOrMoreQuantifier, + ReZeroOrOneQuantifier, + SingleSymbolWildCard, + StrValue, + StrValueManager, + UnboundLenWildCard, +) +from app.translator.platforms.elasticsearch.escape_manager import esql_escape_manager + +AQL_CONTAINER_SPEC_SYMBOLS_MAP = copy.copy(CONTAINER_SPEC_SYMBOLS_MAP) +AQL_CONTAINER_SPEC_SYMBOLS_MAP.update({SingleSymbolWildCard: "_", UnboundLenWildCard: "%"}) + + +class ESQLStrValueManager(StrValueManager): + escape_manager = esql_escape_manager + container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = AQL_CONTAINER_SPEC_SYMBOLS_MAP + container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = AQL_CONTAINER_SPEC_SYMBOLS_MAP + + def from_container_to_str(self, container: StrValue, value_type: str = ValueType.value) -> str: + result = "" + for el in container.split_value: + if isinstance(el, str): + result += self.escape_manager.escape(el, value_type) + elif isinstance(el, BaseSpecSymbol): + if value_type == ValueType.regex_value: + if isinstance(el, SingleSymbolWildCard): + result += "." + continue + if isinstance(el, UnboundLenWildCard): + result += ".*" + continue + + if pattern := self.container_spec_symbols_map.get(type(el)): + result += pattern + + return result + + +esql_str_value_manager = ESQLStrValueManager() From efec67f1abfbe8ffe974ad8a2077c4bed76818f7 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 18 Jun 2024 09:20:50 +0200 Subject: [PATCH 04/40] str value manager --- .../platforms/elasticsearch/renders/esql.py | 10 +++++--- .../elasticsearch/str_value_manager.py | 25 +++---------------- 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index ca89328d..1b012af5 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -17,6 +17,7 @@ ----------------------------------------------------------------- """ from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager @@ -33,28 +34,29 @@ class ESQLFieldValue(SqlFieldValue): def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" - return f'{field} LIKE "*{value}*"' + return f'{field} LIKE "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}*"' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f'{field} LIKE "*{value}?"' + return f'{field} LIKE "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}?"' def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return f'{field} LIKE "{value}%"' + return f'{field} LIKE "{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}%"' def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" - return f'{field} RLIKE "{value}"' + return f'{field} RLIKE \\"{self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=False)}\\"' @render_manager.register class ESQLQueryRender(SqlQueryRender): details: PlatformDetails = elasticsearch_esql_query_details mappings: ElasticSearchMappings = elasticsearch_mappings + comment_symbol = "//" or_token = "or" and_token = "and" diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py index e7727ba4..44ac986a 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -23,24 +23,6 @@ from app.translator.core.str_value_manager import ( CONTAINER_SPEC_SYMBOLS_MAP, BaseSpecSymbol, - ReAnySymbol, - ReCaretSymbol, - ReCommaSymbol, - ReDigitalSymbol, - ReEndOfStrSymbol, - ReHyphenSymbol, - ReLeftCurlyBracket, - ReLeftParenthesis, - ReLeftSquareBracket, - ReOneOrMoreQuantifier, - ReOrOperator, - ReRightCurlyBracket, - ReRightParenthesis, - ReRightSquareBracket, - ReWhiteSpaceSymbol, - ReWordSymbol, - ReZeroOrMoreQuantifier, - ReZeroOrOneQuantifier, SingleSymbolWildCard, StrValue, StrValueManager, @@ -48,14 +30,13 @@ ) from app.translator.platforms.elasticsearch.escape_manager import esql_escape_manager -AQL_CONTAINER_SPEC_SYMBOLS_MAP = copy.copy(CONTAINER_SPEC_SYMBOLS_MAP) -AQL_CONTAINER_SPEC_SYMBOLS_MAP.update({SingleSymbolWildCard: "_", UnboundLenWildCard: "%"}) +ESQL_CONTAINER_SPEC_SYMBOLS_MAP = copy.copy(CONTAINER_SPEC_SYMBOLS_MAP) +ESQL_CONTAINER_SPEC_SYMBOLS_MAP.update({SingleSymbolWildCard: "_", UnboundLenWildCard: "%"}) class ESQLStrValueManager(StrValueManager): escape_manager = esql_escape_manager - container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = AQL_CONTAINER_SPEC_SYMBOLS_MAP - container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = AQL_CONTAINER_SPEC_SYMBOLS_MAP + container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = ESQL_CONTAINER_SPEC_SYMBOLS_MAP def from_container_to_str(self, container: StrValue, value_type: str = ValueType.value) -> str: result = "" From ffff58836edd0b985c0d6ec7ec034db9d70bd139 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 18 Jun 2024 09:20:54 +0200 Subject: [PATCH 05/40] added str value to esql --- .../platforms/elasticsearch/escape_manager.py | 5 +-- .../platforms/elasticsearch/renders/esql.py | 4 +- .../elasticsearch/str_value_manager.py | 40 +++++-------------- 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py index 8fd4d474..9fc25729 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py @@ -7,10 +7,9 @@ class ESQLEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { - ValueType.value: [EscapeDetails(pattern=r'"', escape_symbols=r'\\"')], ValueType.regex_value: [ - EscapeDetails(pattern=r"([$^*+()\[\]{}|.?\-\\])", escape_symbols=r"\\\1"), - EscapeDetails(pattern=r"(')", escape_symbols=r"'\1"), + EscapeDetails(pattern=r'"', escape_symbols=r'\"'), + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\") ], } diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 1b012af5..c547d139 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -39,12 +39,12 @@ def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f'{field} LIKE "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}?"' + return f'{field} LIKE "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}"' def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return f'{field} LIKE "{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}%"' + return f'{field} LIKE "{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}*"' def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py index 44ac986a..2795c786 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -21,41 +21,23 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( - CONTAINER_SPEC_SYMBOLS_MAP, BaseSpecSymbol, - SingleSymbolWildCard, - StrValue, - StrValueManager, - UnboundLenWildCard, + ReDigitalSymbol, + ReWhiteSpaceSymbol, + ReWordSymbol, + StrValueManager ) -from app.translator.platforms.elasticsearch.escape_manager import esql_escape_manager +from app.translator.platforms.elasticsearch.escape_manager import ESQLEscapeManager, esql_escape_manager -ESQL_CONTAINER_SPEC_SYMBOLS_MAP = copy.copy(CONTAINER_SPEC_SYMBOLS_MAP) -ESQL_CONTAINER_SPEC_SYMBOLS_MAP.update({SingleSymbolWildCard: "_", UnboundLenWildCard: "%"}) class ESQLStrValueManager(StrValueManager): - escape_manager = esql_escape_manager - container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = ESQL_CONTAINER_SPEC_SYMBOLS_MAP - - def from_container_to_str(self, container: StrValue, value_type: str = ValueType.value) -> str: - result = "" - for el in container.split_value: - if isinstance(el, str): - result += self.escape_manager.escape(el, value_type) - elif isinstance(el, BaseSpecSymbol): - if value_type == ValueType.regex_value: - if isinstance(el, SingleSymbolWildCard): - result += "." - continue - if isinstance(el, UnboundLenWildCard): - result += ".*" - continue - - if pattern := self.container_spec_symbols_map.get(type(el)): - result += pattern - - return result + escape_manager: ESQLEscapeManager = esql_escape_manager + re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { + "w": ReWordSymbol, + "d": ReDigitalSymbol, + "s": ReWhiteSpaceSymbol, + } esql_str_value_manager = ESQLStrValueManager() From 19a779f80ab5e3cf287d87a2d677e1c80298c9f5 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 18 Jun 2024 09:20:59 +0200 Subject: [PATCH 06/40] linter --- .../app/translator/platforms/elasticsearch/__init__.py | 4 ++-- .../app/translator/platforms/elasticsearch/const.py | 6 +++--- .../translator/platforms/elasticsearch/escape_manager.py | 6 +++--- .../translator/platforms/elasticsearch/renders/esql.py | 4 ++-- .../platforms/elasticsearch/renders/esql_rule.py | 8 +++----- .../platforms/elasticsearch/str_value_manager.py | 5 +---- 6 files changed, 14 insertions(+), 19 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py index 3c53e3cf..6c1fca9e 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py @@ -4,7 +4,7 @@ from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elasticsearch_cti import ElasticsearchCTI # noqa: F401 +from app.translator.platforms.elasticsearch.renders.esql import ESQLQueryRender # noqa: F401 +from app.translator.platforms.elasticsearch.renders.esql_rule import ESQLRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.kibana import KibanaRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.xpack_watcher import XPackWatcherRuleRender # noqa: F401 -from app.translator.platforms.elasticsearch.renders.esql import ESQLQueryRender # noqa: F401 -from app.translator.platforms.elasticsearch.renders.esql_rule import ESQLRuleRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/elasticsearch/const.py b/uncoder-core/app/translator/platforms/elasticsearch/const.py index 9a93c31e..a87d5e84 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/const.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/const.py @@ -18,7 +18,7 @@ _ELASTALERT_LUCENE_RULE, _ELASTIC_WATCHER_RULE, _ELASTIC_ESQL_QUERY, - _ELASTIC_ESQL_RULE + _ELASTIC_ESQL_RULE, } ELASTICSEARCH_LUCENE_QUERY_DETAILS = { @@ -219,5 +219,5 @@ "type": "esql", "language": "esql", "query": "", - "actions": [] -} \ No newline at end of file + "actions": [], +} diff --git a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py index 9fc25729..9b0e2218 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py @@ -8,9 +8,9 @@ class ESQLEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { ValueType.regex_value: [ - EscapeDetails(pattern=r'"', escape_symbols=r'\"'), - EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\") - ], + EscapeDetails(pattern=r'"', escape_symbols=r"\""), + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), + ] } diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index c547d139..f1f37d1a 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -22,8 +22,8 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details +from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings from app.translator.platforms.elasticsearch.str_value_manager import esql_str_value_manager @@ -65,4 +65,4 @@ class ESQLQueryRender(SqlQueryRender): def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) if str(log_source_signature) else "*" - return f"FROM {table} metadata _id, _version, _index |" \ No newline at end of file + return f"FROM {table} metadata _id, _version, _index |" diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py index 21ac4f36..8802456e 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -19,16 +19,16 @@ import copy import json from typing import Optional, Union + from app.translator.core.mapping import SourceMapping from app.translator.core.mitre import MitreConfig from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager +from app.translator.platforms.elasticsearch.const import ESQL_RULE, elasticsearch_esql_rule_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -from app.translator.platforms.elasticsearch.const import elasticsearch_esql_rule_details, ESQL_RULE from app.translator.platforms.elasticsearch.renders.esql import ESQLFieldValue, ESQLQueryRender - _AUTOGENERATED_TEMPLATE = "Autogenerated ESQL Rule" @@ -36,14 +36,12 @@ class ESQLRuleFieldValue(ESQLFieldValue): details: PlatformDetails = elasticsearch_esql_rule_details - @render_manager.register class ESQLRuleRender(ESQLQueryRender): details: PlatformDetails = elasticsearch_esql_rule_details mappings: ElasticSearchMappings = elasticsearch_mappings mitre: MitreConfig = MitreConfig() - def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): return [] @@ -102,4 +100,4 @@ def finalize_query( if not_supported_functions: rendered_not_supported = self.render_not_supported_functions(not_supported_functions) return rule_str + rendered_not_supported - return rule_str \ No newline at end of file + return rule_str diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py index 2795c786..d0eb2f17 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -16,21 +16,18 @@ limitations under the License. ----------------------------------------------------------------- """ -import copy from typing import ClassVar -from app.translator.core.custom_types.values import ValueType from app.translator.core.str_value_manager import ( BaseSpecSymbol, ReDigitalSymbol, ReWhiteSpaceSymbol, ReWordSymbol, - StrValueManager + StrValueManager, ) from app.translator.platforms.elasticsearch.escape_manager import ESQLEscapeManager, esql_escape_manager - class ESQLStrValueManager(StrValueManager): escape_manager: ESQLEscapeManager = esql_escape_manager re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { From d7be69d9f23bb9beec4c9b1c90f13db719bdd86f Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:02:55 +0200 Subject: [PATCH 07/40] update modifiers process --- .../platforms/elasticsearch/escape_manager.py | 3 ++ .../platforms/elasticsearch/renders/esql.py | 36 ++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py index 9b0e2218..e1d78880 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py @@ -7,6 +7,9 @@ class ESQLEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [ + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\") + ], ValueType.regex_value: [ EscapeDetails(pattern=r'"', escape_symbols=r"\""), EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index f1f37d1a..2d4f1298 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -16,10 +16,13 @@ limitations under the License. ----------------------------------------------------------------- """ +from curses.ascii import isdigit +from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details @@ -31,25 +34,50 @@ class ESQLFieldValue(SqlFieldValue): details: PlatformDetails = elasticsearch_esql_query_details str_value_manager = esql_str_value_manager + @staticmethod + def _make_case_insensitive(value: str) -> str: + container: list[str] = [] + for v in value: + if v.isalpha(): + container.append(f'[{v.upper()}{v.lower()}]') + else: + container.append(v) + return ''.join(container) + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f'"{value}"' + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field.endswith('.text'): + return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" - return f'{field} LIKE "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}*"' + return f'contains({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field.endswith('.text'): + return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f'{field} LIKE "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}"' + return f'ends_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})' def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field.endswith('.text'): + return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return f'{field} LIKE "{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}*"' + return f'starts_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})' def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" - return f'{field} RLIKE \\"{self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=False)}\\"' + pre_processed_value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=False) + if isinstance(pre_processed_value, str): + value = self._make_case_insensitive(pre_processed_value) + else: + value = pre_processed_value + return f'{field} rlike "{value}"' @render_manager.register From a8580fa273897b38e6c0a22d4207687efc35f5a2 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:47:52 +0200 Subject: [PATCH 08/40] added value wrap --- .../platforms/elasticsearch/escape_manager.py | 4 ++- .../platforms/elasticsearch/renders/esql.py | 36 ++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py index e1d78880..193365a2 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py @@ -8,11 +8,13 @@ class ESQLEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { ValueType.value: [ - EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\") + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), + EscapeDetails(pattern=r'"', escape_symbols=r"\"") ], ValueType.regex_value: [ EscapeDetails(pattern=r'"', escape_symbols=r"\""), EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), + EscapeDetails(pattern=r'([_!@#$%^&*=+()\[\]{}|;:\'",.<>?/`~\-\s\\])', escape_symbols=r"\\\\\1") ] } diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 2d4f1298..abac539f 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -16,23 +16,21 @@ limitations under the License. ----------------------------------------------------------------- """ -from curses.ascii import isdigit from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -from app.translator.platforms.elasticsearch.str_value_manager import esql_str_value_manager +from app.translator.platforms.elasticsearch.str_value_manager import ESQLStrValueManager, esql_str_value_manager class ESQLFieldValue(SqlFieldValue): details: PlatformDetails = elasticsearch_esql_query_details - str_value_manager = esql_str_value_manager + str_value_manager: ESQLStrValueManager = esql_str_value_manager @staticmethod def _make_case_insensitive(value: str) -> str: @@ -48,12 +46,34 @@ def _make_case_insensitive(value: str) -> str: def _wrap_str_value(value: str) -> str: return f'"{value}"' + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" + return f"{field} = {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + + def less_modifier(self, field: str, value: Union[int, str]) -> str: + return f"{field} < {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + + def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + return f"{field} <= {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + + def greater_modifier(self, field: str, value: Union[int, str]) -> str: + return f"{field} > {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + + def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + return f"{field} >= {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" + return f"{field} != {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if field.endswith('.text'): - return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" - return f'contains({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})' + if field.endswith('.text'): + return self.regex_modifier(field=field, value=value) + return f'{field} like "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}*"' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith('.text'): @@ -77,7 +97,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: value = self._make_case_insensitive(pre_processed_value) else: value = pre_processed_value - return f'{field} rlike "{value}"' + return f'{field} rlike ".*{value}.*"' @render_manager.register From cbfb812f10592c9c60c9712717e15e87a9dbe356 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Thu, 20 Jun 2024 08:53:20 +0200 Subject: [PATCH 09/40] merge prod --- uncoder-core/app/routers/translate.py | 8 +- .../app/translator/core/context_vars.py | 4 + .../app/translator/core/custom_types/time.py | 9 + uncoder-core/app/translator/core/mapping.py | 8 +- uncoder-core/app/translator/core/parser.py | 1 + uncoder-core/app/translator/core/render.py | 58 +++- .../app/translator/core/render_cti.py | 1 - .../app/translator/core/str_value_manager.py | 1 + .../palo_alto_cortex/aws_cloudtrail.yml | 36 +++ .../platforms/palo_alto_cortex/aws_eks.yml | 25 ++ .../azure_aadnoninteractiveusersigninlogs.yml | 16 + .../palo_alto_cortex/azure_azureactivity.yml | 33 +++ .../palo_alto_cortex/azure_azuread.yml | 34 +++ .../platforms/palo_alto_cortex/azure_m365.yml | 34 +++ .../palo_alto_cortex/azure_signinlogs.yml | 46 +++ .../platforms/palo_alto_cortex/default.yml | 12 +- .../platforms/palo_alto_cortex/dns.yml | 13 + .../platforms/palo_alto_cortex/okta_okta.yml | 3 +- .../platforms/palo_alto_cortex/proxy.yml | 13 +- .../platforms/palo_alto_cortex/webserver.yml | 5 + .../palo_alto_cortex/windows_application.yml | 26 +- .../palo_alto_cortex/windows_image_load.yml | 3 +- .../windows_network_connection.yml | 6 +- .../palo_alto_cortex/windows_pipe_created.yml | 4 +- .../palo_alto_cortex/windows_powershell.yml | 15 +- .../windows_process_access.yml | 19 +- .../windows_process_termination.yml | 13 + .../palo_alto_cortex/windows_security.yml | 280 +++++++++--------- .../palo_alto_cortex/windows_sysmon.yml | 95 +++--- .../palo_alto_cortex/windows_system.yml | 24 +- .../mappings/platforms/qradar/default.yml | 31 +- .../mappings/platforms/qradar/firewall.yml | 1 + .../mappings/platforms/qradar/proxy.yml | 29 +- .../platforms/qradar/windows_image_load.yml | 10 +- .../qradar/windows_process_termination.yml | 16 + .../platforms/qradar/windows_security.yml | 61 +++- .../platforms/sigma/aws_cloudtrail.yml | 2 +- .../platforms/sigma/azure_azureactivity.yml | 2 +- .../platforms/sigma/azure_azuread.yml | 2 +- .../mappings/platforms/sigma/azure_m365.yml | 2 +- .../platforms/sigma/windows_pipe_created.yml | 16 + .../sigma/windows_registry_event.yml | 2 +- .../app/translator/platforms/__init__.py | 9 +- .../platforms/athena/renders/athena.py | 6 +- .../translator/platforms/base/aql/const.py | 4 +- .../platforms/base/aql/renders/aql.py | 6 +- .../platforms/base/aql/str_value_manager.py | 1 + .../platforms/base/aql/tokenizer.py | 1 + .../platforms/base/lucene/renders/lucene.py | 3 +- .../base/lucene/str_value_manager.py | 1 + .../platforms/base/lucene/tokenizer.py | 1 + .../platforms/base/spl/renders/spl.py | 2 +- .../platforms/base/sql/renders/sql.py | 6 +- .../platforms/chronicle/parsers/chronicle.py | 1 - .../platforms/chronicle/renders/chronicle.py | 2 +- .../crowdstrike/parsers/crowdstrike.py | 1 + .../crowdstrike/renders/crowdstrike.py | 10 +- .../platforms/elasticsearch/escape_manager.py | 6 +- .../elasticsearch/parsers/detection_rule.py | 1 - .../elasticsearch/renders/detection_rule.py | 5 +- .../elasticsearch/renders/elast_alert.py | 2 +- .../platforms/elasticsearch/renders/esql.py | 17 +- .../platforms/elasticsearch/renders/kibana.py | 1 + .../elasticsearch/renders/xpack_watcher.py | 1 + .../forti_siem/renders/forti_siem_rule.py | 8 +- .../platforms/forti_siem/str_value_manager.py | 3 + .../platforms/hunters/renders/hunters.py | 6 +- .../renders/logrhythm_axon_query.py | 9 +- .../renders/logrhythm_axon_rule.py | 1 + .../platforms/logscale/parsers/logscale.py | 1 - .../logscale/parsers/logscale_alert.py | 1 - .../platforms/logscale/renders/logscale.py | 15 +- .../logscale/renders/logscale_alert.py | 1 + .../microsoft/parsers/microsoft_sentinel.py | 1 - .../parsers/microsoft_sentinel_rule.py | 1 - .../microsoft/renders/microsoft_defender.py | 6 +- .../renders/microsoft_defender_cti.py | 1 + .../microsoft/renders/microsoft_sentinel.py | 14 +- .../renders/microsoft_sentinel_rule.py | 1 + .../opensearch/renders/opensearch.py | 1 + .../opensearch/renders/opensearch_rule.py | 2 +- .../palo_alto/renders/cortex_xsiam.py | 36 ++- .../platforms/palo_alto/str_value_manager.py | 1 + .../platforms/qradar/renders/qradar.py | 1 + .../platforms/sigma/escape_manager.py | 2 +- .../platforms/sigma/models/compiler.py | 1 + .../platforms/sigma/models/modifiers.py | 5 +- .../platforms/sigma/parsers/sigma.py | 30 +- .../platforms/sigma/renders/sigma.py | 2 +- .../platforms/sigma/str_value_manager.py | 3 +- .../translator/platforms/sigma/tokenizer.py | 7 +- .../platforms/splunk/renders/splunk.py | 9 +- .../platforms/splunk/renders/splunk_alert.py | 1 + uncoder-core/app/translator/tools/utils.py | 9 + 94 files changed, 904 insertions(+), 381 deletions(-) create mode 100644 uncoder-core/app/translator/core/context_vars.py create mode 100644 uncoder-core/app/translator/core/custom_types/time.py create mode 100644 uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_eks.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_aadnoninteractiveusersigninlogs.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_azureactivity.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_azuread.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_m365.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_signinlogs.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_termination.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/qradar/windows_process_termination.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/sigma/windows_pipe_created.yml diff --git a/uncoder-core/app/routers/translate.py b/uncoder-core/app/routers/translate.py index 7acdaaee..009cab03 100644 --- a/uncoder-core/app/routers/translate.py +++ b/uncoder-core/app/routers/translate.py @@ -1,6 +1,7 @@ from fastapi import APIRouter, Body from app.models.translation import InfoMessage, OneTranslationData, Platform, TranslatorPlatforms +from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.cti_translator import CTITranslator from app.translator.translator import Translator @@ -15,7 +16,9 @@ def translate_one( source_platform_id: str = Body(..., embed=True), target_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True), + return_only_first_query: bool = False, ) -> OneTranslationData: + return_only_first_query_ctx_var.set(return_only_first_query) status, data = translator.translate_one(text=text, source=source_platform_id, target=target_platform_id) if status: return OneTranslationData(status=status, translation=data, target_platform_id=target_platform_id) @@ -27,8 +30,11 @@ def translate_one( @st_router.post("/translate/all", tags=["translator"], description="Generate all translations") @st_router.post("/translate/all/", include_in_schema=False) def translate_all( - source_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True) + source_platform_id: str = Body(..., embed=True), + text: str = Body(..., embed=True), + return_only_first_query: bool = False, ) -> list[OneTranslationData]: + return_only_first_query_ctx_var.set(return_only_first_query) result = translator.translate_all(text=text, source=source_platform_id) translations = [] for platform_result in result: diff --git a/uncoder-core/app/translator/core/context_vars.py b/uncoder-core/app/translator/core/context_vars.py new file mode 100644 index 00000000..591883d8 --- /dev/null +++ b/uncoder-core/app/translator/core/context_vars.py @@ -0,0 +1,4 @@ +from contextvars import ContextVar + +return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False) +"""Set to True to return only first query if rendered multiple options""" diff --git a/uncoder-core/app/translator/core/custom_types/time.py b/uncoder-core/app/translator/core/custom_types/time.py new file mode 100644 index 00000000..1d5f15b8 --- /dev/null +++ b/uncoder-core/app/translator/core/custom_types/time.py @@ -0,0 +1,9 @@ +from app.translator.tools.custom_enum import CustomEnum + + +class TimeFrameType(CustomEnum): + years = "years" + months = "months" + days = "days" + hours = "hours" + minutes = "minutes" diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 0ecccbc1..bdab5f6d 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -20,6 +20,10 @@ def is_suitable(self, *args, **kwargs) -> bool: def __str__(self) -> str: raise NotImplementedError("Abstract method") + @property + def default_source(self) -> dict: + return self._default_source + class FieldMapping: def __init__(self, generic_field_name: str, platform_field_name: str): @@ -72,7 +76,7 @@ def __init__( source_id: str, log_source_signature: _LogSourceSignatureType = None, fields_mapping: Optional[FieldsMapping] = None, - raw_log_fields: Optional[list] = None, + raw_log_fields: Optional[dict] = None, ): self.source_id = source_id self.log_source_signature = log_source_signature @@ -103,7 +107,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]: continue field_mappings_dict = mapping_dict.get("field_mapping", {}) - raw_log_fields = mapping_dict.get("raw_log_fields", []) + raw_log_fields = mapping_dict.get("raw_log_fields", {}) field_mappings_dict.update({field: field for field in raw_log_fields}) fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict) self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping) diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 791734be..7cc10ec1 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -15,6 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ + import re from abc import ABC, abstractmethod from typing import Union diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 81dec670..055ce889 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -16,11 +16,13 @@ limitations under the License. ----------------------------------------------------------------- """ + from abc import ABC, abstractmethod from collections.abc import Callable -from typing import Optional, Union +from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -164,7 +166,14 @@ class QueryRender(ABC): is_single_line_comment: bool = False unsupported_functions_text = "Unsupported functions were excluded from the result query:" - platform_functions: PlatformFunctions = PlatformFunctions() + platform_functions: PlatformFunctions = None + + def __init__(self): + self.init_platform_functions() + + def init_platform_functions(self) -> None: + self.platform_functions = PlatformFunctions() + self.platform_functions.platform_query_render = self def render_not_supported_functions(self, not_supported_functions: list) -> str: line_template = f"{self.comment_symbol} " if self.comment_symbol and self.is_single_line_comment else "" @@ -191,19 +200,19 @@ class PlatformQueryRender(QueryRender): field_value_map = BaseQueryFieldValue(or_token=or_token) - query_pattern = "{table} {query} {functions}" - raw_log_field_pattern: str = None + raw_log_field_pattern_map: ClassVar[dict[str, str]] = None def __init__(self): + super().__init__() self.operator_map = { LogicalOperatorType.AND: f" {self.and_token} ", LogicalOperatorType.OR: f" {self.or_token} ", LogicalOperatorType.NOT: f" {self.not_token} ", } - def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 - if str(log_source_signature): - return f"{log_source_signature!s} {self.and_token}" + def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002 + if log_source_signature and str(log_source_signature): + return f"{log_source_signature} {self.and_token}" return "" def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions: @@ -271,6 +280,10 @@ def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> query = f"{query}\n\n{query_meta_info}" return query + @staticmethod + def _finalize_search_query(query: str) -> str: + return query + def finalize_query( self, prefix: str, @@ -282,7 +295,8 @@ def finalize_query( *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: - query = self.query_pattern.format(prefix=prefix, query=query, functions=functions).strip() + parts = filter(lambda s: bool(s), map(str.strip, [prefix, self._finalize_search_query(query), functions])) + query = " ".join(parts) query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) if not_supported_functions: rendered_not_supported = self.render_not_supported_functions(not_supported_functions) @@ -323,7 +337,23 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer) prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info ) + def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: + if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type): + return raw_log_field_pattern.format(field=field) + + def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[list]: + if isinstance(field, list): + prefix_list = [] + for f in field: + if _prefix_list := self.process_raw_log_field_prefix(field=f, source_mapping=source_mapping): + prefix_list.extend(_prefix_list) + return prefix_list + if raw_log_field_type := source_mapping.raw_log_fields.get(field): + return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)] + def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str: + if self.raw_log_field_pattern_map is None: + return "" defined_raw_log_fields = [] for field in fields: mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=field.source_name) @@ -334,11 +364,11 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap ) if not mapped_field and self.is_strict_mapping: raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name) - if mapped_field not in source_mapping.raw_log_fields: - continue - field_prefix = self.raw_log_field_pattern.format(field=mapped_field) - defined_raw_log_fields.append(field_prefix) - return "\n".join(set(defined_raw_log_fields)) + if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping): + for prefix in prefix_list: + if prefix not in defined_raw_log_fields: + defined_raw_log_fields.append(prefix) + return "\n".join(defined_raw_log_fields) def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} @@ -368,6 +398,8 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue meta_info=query_container.meta_info, source_mapping=source_mapping, ) + if return_only_first_query_ctx_var.get() is True: + return finalized_query queries_map[source_mapping.source_id] = finalized_query if not queries_map and errors: raise errors[0] diff --git a/uncoder-core/app/translator/core/render_cti.py b/uncoder-core/app/translator/core/render_cti.py index 52a65ea6..20bfb7bf 100644 --- a/uncoder-core/app/translator/core/render_cti.py +++ b/uncoder-core/app/translator/core/render_cti.py @@ -17,7 +17,6 @@ ----------------------------------------------------------------- """ - from app.translator.core.models.iocs import IocsChunkValue from app.translator.core.models.platform_details import PlatformDetails diff --git a/uncoder-core/app/translator/core/str_value_manager.py b/uncoder-core/app/translator/core/str_value_manager.py index cd7523c0..74a9f532 100644 --- a/uncoder-core/app/translator/core/str_value_manager.py +++ b/uncoder-core/app/translator/core/str_value_manager.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import ClassVar, Optional, TypeVar, Union from app.translator.core.custom_types.values import ValueType diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml new file mode 100644 index 00000000..980f2125 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml @@ -0,0 +1,36 @@ +platform: Palo Alto XSIAM +source: aws_cloudtrail + + +default_log_source: + dataset: amazon_aws_raw + +field_mapping: + eventSource: eventSource + eventName: eventName + errorCode: errorCode + errorMessage: errorMessage + eventType: eventType + requestParameters: requestParameters + responseElements: responseElements + status: status + terminatingRuleId: terminatingRuleId + userAgent: userAgent + AdditionalEventData.MFAUsed: additionalEventData.MFAUsed + + +raw_log_fields: + additionalEventData.MFAUsed: object + requestParameters.ipPermissions.items.ipRanges.items.cidrIP: object + requestParameters.ipPermissions.items.ipRanges.items.fromPort: object + requestParameters.attribute: object + requestParameters.userData: list + responseElements.ConsoleLogin: object + responseElements.pendingModifiedValues.masterUserPassword: object + responseElements.publiclyAccessible: object + userIdentity.arn: object + userIdentity.principalId: object + userIdentity.sessionContext.sessionIssuer.type: object + userIdentity.type: object + userIdentity.userName: object + requestParameters.publiclyAccessible: object \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_eks.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_eks.yml new file mode 100644 index 00000000..e7ba2c05 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_eks.yml @@ -0,0 +1,25 @@ +platform: Palo Alto XSIAM +source: aws_eks + + +default_log_source: + dataset: amazon_aws_raw + +field_mapping: + aws_node_type: aws_node_type + requestURI: requestURI + stage: stage + verb: verb + + +raw_log_fields: + annotations.authorization.k8s.io\/decision: object + annotations.podsecuritypolicy.policy.k8s.io\/admit-policy: object + objectRef.namespace: object + objectRef.resource: object + objectRef.subresource: object + requestObject.rules.resources: object + requestObject.rules.verbs: object + requestObject.spec.containers.image: object + user.groups: object + user.username: object \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_aadnoninteractiveusersigninlogs.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_aadnoninteractiveusersigninlogs.yml new file mode 100644 index 00000000..cd489ccb --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_aadnoninteractiveusersigninlogs.yml @@ -0,0 +1,16 @@ +platform: Palo Alto XSIAM +source: azure_aadnoninteractiveusersigninlogs + + +default_log_source: + dataset: msft_azure_raw + +field_mapping: + UserAgent: properties.userAgent + Type: properties.type + AuthenticationProcessingDetails: properties.authenticationProcessingDetails + +raw_log_fields: + properties.userAgent: object + properties.type: object + properties.authenticationProcessingDetails: list \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_azureactivity.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_azureactivity.yml new file mode 100644 index 00000000..b6605a61 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_azureactivity.yml @@ -0,0 +1,33 @@ +platform: Palo Alto XSIAM +source: azure_azureactivity + + +default_log_source: + dataset: msft_azure_raw + +field_mapping: + ActivityStatus: properties.activityStatus + ActivityStatusValue: properties.activityStatusValue + ActivitySubstatusValue: properties.activitySubstatusValue + Authorization: properties.authorization + Category: properties.category + CategoryValue: properties.categoryValue + OperationName: properties.operationName + OperationNameValue: oproperties.perationNameValue + ResourceId: properties.resourceId + ResourceProviderValue: properties.resourceProviderValue + Type: properties.type + operationName: properties.operationName + +raw_log_fields: + properties.activityStatus: object + properties.activityStatusValue: object + properties.activitySubstatusValue: object + properties.authorization: object + properties.category: object + properties.categoryValue: object + properties.operationName: object + properties.operationNameValue: object + properties.resourceId: object + properties.resourceProviderValue: object + properties.type: object \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_azuread.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_azuread.yml new file mode 100644 index 00000000..c05ce310 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_azuread.yml @@ -0,0 +1,34 @@ +platform: Palo Alto XSIAM +source: azure_azuread + + +default_log_source: + dataset: msft_azure_raw + +field_mapping: + ActivityDisplayName: properties.activityDisplayName + Category: properties.category + LoggedByService: properties.loggedByService + Result: properties.result + OperationName: properties.operationName + TargetResources: properties.targetResources + AADOperationType: properties.AADOperationType + InitiatedBy: properties.initiatedBy + ResultReason: properties.resultReason + Status: properties.status + #Status.errorCode: properties.status_errorCode + UserAgent: properties.userAgent + +raw_log_fields: + properties.activityDisplayName: object + properties.category: object + properties.loggedByService: object + properties.result: object + properties.operationName: object + properties.targetResources: object + properties.AADOperationType: object + properties.initiatedBy: object + properties.resultReason: object + properties.status: object + properties.status_errorCode: object + properties.userAgent: object \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_m365.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_m365.yml new file mode 100644 index 00000000..ea4cfecf --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_m365.yml @@ -0,0 +1,34 @@ +platform: Palo Alto XSIAM +source: azure_m365 + + +default_log_source: + dataset: msft_azure_raw + +field_mapping: + ClientInfoString: properties.clientInfoString + LogonError: properties.logonError + ModifiedProperties: properties.modifiedProperties + OfficeObjectId: properties.officeObjectId + OfficeWorkload: properties.officeWorkload + Operation: properties.operation + Parameters: properties.parameters + RecordType: properties.recordType + ResultStatus: properties.resultStatus + SourceFileExtension: properties.sourceFileExtension + SourceFileName: properties.sourceFileName + UserAgent: properties.userAgent + +raw_log_fields: + properties.clientInfoString: object + properties.logonError: object + properties.modifiedProperties: object + properties.officeObjectId: object + properties.officeWorkload: object + properties.operation: object + properties.parameters: object + properties.recordType: object + properties.resultStatus: object + properties.sourceFileExtension: object + properties.sourceFileName: object + properties.userAgent: object \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_signinlogs.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_signinlogs.yml new file mode 100644 index 00000000..b5b84cde --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_signinlogs.yml @@ -0,0 +1,46 @@ +platform: Palo Alto XSIAM +source: azure_signinlogs + + +default_log_source: + dataset: msft_azure_raw + +field_mapping: + AppDisplayName: properties.appDisplayName + AppId: properties.appId + AuthenticationRequirement: properties.authenticationRequirement + Category: properties.category + ConditionalAccessStatus: properties.conditionalAccessStatus + DeviceDetail: properties.deviceDetail + IsInteractive: properties.isInteractive + NetworkLocationDetails: properties.networkLocationDetails + ResourceDisplayName: properties.resourceDisplayName + ResourceIdentity: properties.resourceIdentity + ResultDescription: properties.resultDescription + ResultType: properties.resultType + Status.errorCode: properties.status.errorCode + Status: properties.status + Status.failureReason: properties.status.failureReason + TokenIssuerType: properties.tokenIssuerType + UserAgent: properties.userAgent + UserPrincipalName: properties.userPrincipalName + +raw_log_fields: + properties.appDisplayName: object + properties.appId: object + properties.authenticationRequirement: object + properties.category: object + properties.conditionalAccessStatus: object + properties.deviceDetail: object + properties.isInteractive: object + properties.networkLocationDetails: object + properties.resourceDisplayName: object + properties.resourceIdentity: object + properties.resultDescription: object + properties.resultType: object + properties.status.errorCode: object + properties.status: object + properties.status.failureReason: object + properties.tokenIssuerType: object + properties.userAgent: object + properties.userPrincipalName: object \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml index 30995299..f6b25023 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml @@ -35,6 +35,7 @@ field_mapping: DestinationIp: xdm.target.ipv4 dst-port: xdm.target.port DestinationPort: xdm.target.port + destinationPort: xdm.target.port src-bytes: xdm.source.sent_bytes dst-bytes: xdm.target.sent_bytes src-hostname: xdm.source.host.hostname @@ -75,14 +76,15 @@ field_mapping: NewTargetUserName: xdm.target.user.username OldTargetUserName: xdm.target.user.username UserPrincipalName: xdm.source.user.username - DestAddress: xdm.target.ipv4 + SubjectAccountName: xdm.source.user.username SubjectUserName: xdm.source.user.username SubjectUserSid: xdm.source.user.identifier SourceAddr: xdm.source.ipv4 SourceAddress: xdm.source.ipv4 TargetSid: xdm.target.user.identifier TargetUserName: xdm.target.user.username + SourceUserName: xdm.source.user.username ParentProcessName: xdm.source.process.executable.path client.user.full_name: xdm.target.user.username source.user.full_name: xdm.source.user.username @@ -115,3 +117,11 @@ field_mapping: http.method: xdm.network.http.method method: xdm.network.http.method notice.user_agent: xdm.network.http.browser + hasIdentity: xdm.source.user.identity_type + ComputerName: xdm.source.host.hostname + ExternalSeverity: xdm.alert.severity + SourceMAC: xdm.source.host.mac_addresses + DestinationMAC: xdm.target.host.mac_addresses + SourceOS: xdm.source.host.os + DestinationOS: xdm.target.host.os + url_category: xdm.network.http.url_category diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml new file mode 100644 index 00000000..e489fd50 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml @@ -0,0 +1,13 @@ +platform: Palo Alto XSIAM +source: dns + +default_log_source: + datamodel: datamodel + +field_mapping: + dns-query: xdm.network.dns.dns_question.name + dns-answer: xdm.network.dns.dns_resource_record.value + #dns-record: dns-record + dns_query_name: xdm.network.dns.dns_question.name + QueryName: xdm.network.dns.dns_question.name + query: xdm.network.dns.dns_question.name \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/okta_okta.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/okta_okta.yml index 6700e0a0..c0ed1066 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/okta_okta.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/okta_okta.yml @@ -7,4 +7,5 @@ default_log_source: dataset: okta_okta_raw field_mapping: - eventType: xdm.event.type \ No newline at end of file + eventType: xdm.event.type + eventtype: xdm.event.type \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/proxy.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/proxy.yml index 9cee722e..c546dc4e 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/proxy.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/proxy.yml @@ -6,9 +6,18 @@ default_log_source: field_mapping: c-uri: xdm.network.http.url - c-useragent: xdm.source.user_agent + c-useragent: xdm.network.http.browser cs-method: xdm.network.http.method cs-bytes: xdm.target.sent_bytes c-uri-query: xdm.network.http.url cs-referrer: xdm.network.http.referrer - sc-status: xdm.network.http.response_code \ No newline at end of file + sc-status: xdm.network.http.response_code + cs-host: xdm.network.http.domain + cs-uri-query: xdm.network.http.url + cs-cookie-vars: xdm.network.http.http_header.value + c-uri-extension: xdm.network.http.url + cs-cookie: xdm.network.http.http_header.value + #cs-version: cs-version + r-dns: xdm.network.http.domain + post-body: xdm.network.http.http_header.value + url_category: xdm.network.http.url_category \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/webserver.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/webserver.yml index 49a58521..7a1eaa84 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/webserver.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/webserver.yml @@ -12,3 +12,8 @@ field_mapping: c-uri-query: xdm.network.http.url cs-referrer: xdm.network.http.referrer sc-status: xdm.network.http.response_code + cs-uri-stem: xdm.network.http.url + cs-uri-query: xdm.network.http.url + c-uri-path: xdm.network.http.url + uri_path: xdm.network.http.url + cs-uri: xdm.network.http.url diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml index 71143f9c..d40073fd 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml @@ -9,16 +9,16 @@ field_mapping: Provider_Name: provider_name raw_log_fields: - - src_ip - - source - - additional_information - - EventData - - Channel - - statement - - Faulting application path - - object_name - - class_type - - action_id - - Data - - Message - - Level + src_ip: regex + source: regex + additional_information: regex + EventData: regex + Channel: regex + statement: regex + Faulting application path: regex + object_name: regex + class_type: regex + action_id: regex + Data: regex + Message: regex + Level: regex diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_image_load.yml index 23b288b3..69a100ec 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_image_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_image_load.yml @@ -27,4 +27,5 @@ field_mapping: ParentIntegrityLevel: causality_actor_process_integrity_level ParentLogonId: causality_actor_process_logon_id ParentProduct: causality_actor_process_signature_product - ParentCompany: causality_actor_process_signature_vendor \ No newline at end of file + ParentCompany: causality_actor_process_signature_vendor + Signed: actor_process_signature_status #Signature status of the process: Signed = 1 SignedInvalid = 2 Unsigned = 3 FailedToObtain = 4 WeakHash = 5, where the MD5 is used as the hash algorithm. Unsupported = 6, which means the signature was not calculated. InvalidCVE2020_0601 = 7, which means the executable is malicious and is trying to exploit the windows vulnerability CVE2020-0601. Deleted = 8, which means that the file was deleted by the time the agent tried to calculate the signature. \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_network_connection.yml index 11d75858..9c535767 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_network_connection.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_network_connection.yml @@ -51,4 +51,8 @@ field_mapping: ParentIntegrityLevel: causality_actor_process_integrity_level ParentLogonId: causality_actor_process_logon_id ParentProduct: causality_actor_process_signature_product - ParentCompany: causality_actor_process_signature_vendor \ No newline at end of file + ParentCompany: causality_actor_process_signature_vendor + + +raw_log_fields: + Initiated: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_pipe_created.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_pipe_created.yml index 2e7ea732..8deb0974 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_pipe_created.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_pipe_created.yml @@ -8,5 +8,5 @@ field_mapping: EventID: action_evtlog_event_id raw_log_fields: - - PipeName - - Image \ No newline at end of file + PipeName: regex + Image: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml index 6af38835..41ed1439 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml @@ -10,10 +10,11 @@ field_mapping: raw_log_fields: - - CommandLine - - ScriptBlockText - - Payload - - HostApplication - - ContextInfo - - HostName - - EngineVersion \ No newline at end of file + CommandLine: regex + ScriptBlockText: regex + Payload: regex + HostApplication: regex + ContextInfo: regex + HostName: regex + EngineVersion: regex + Path: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_access.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_access.yml index 47a1033e..ab559df0 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_access.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_access.yml @@ -6,14 +6,15 @@ default_log_source: field_mapping: User: action_process_username + SourceUser: action_process_username raw_log_fields: - - SourceProcessGUID - - SourceProcessId - - SourceThreadId - - SourceImage - - TargetProcessGUID - - TargerProcessId - - TargetImage - - GrantedAccess - - CallTrace \ No newline at end of file + SourceProcessGUID: regex + SourceProcessId: regex + SourceThreadId: regex + SourceImage: regex + TargetProcessGUID: regex + TargerProcessId: regex + TargetImage: regex + GrantedAccess: regex + CallTrace: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_termination.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_termination.yml new file mode 100644 index 00000000..731d6b8e --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_process_termination.yml @@ -0,0 +1,13 @@ +platform: Palo Alto XSIAM +source: windows_process_termination + +log_source: + preset: xdr_process + +default_log_source: + preset: xdr_process + +field_mapping: + Image: action_process_image_path + ProcessId: action_process_os_pid + ProcessGuid: ProcessGuid \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_security.yml index a2abf004..59a56f71 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_security.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_security.yml @@ -7,142 +7,146 @@ default_log_source: field_mapping: EventID: action_evtlog_event_id Provider_Name: provider_name - + SubjectAccountName: actor_effective_username + raw_log_fields: - - ParentImage - - AccessMask - - AccountName - - AllowedToDelegateTo - - AttributeLDAPDisplayName - - AuditPolicyChanges - - AuthenticationPackageName - - CallingProcessName - - Channel - - ComputerName - - EventType - - FailureReason - - FileName - - GrantedAccess - - Hashes - - HiveName - - IpAddress - - IpPort - - KeyLength - - LogonProcessName - - LogonType - - LinkName - - ProcessId - - PublishURLs - - ElevatedToken - - MemberName - - MemberSid - - NewProcessName - - ObjectClass - - ObjectName - - ObjectType - - ObjectValueName - - Path - - CommandLine - - OldUacValue - - CertIssuerName - - SubStatus - - DisplayName - - TaskContent - - ServiceSid - - CertThumbprint - - ClassName - - NotificationPackageName - - NewSd - - TestSigning - - TargetInfo - - ParentProcessId - - AccessList - - GroupMembership - - FilterName - - ChangeType - - LayerName - - ServiceAccount - - ClientProcessId - - AttributeValue - - SessionName - - TaskName - - ObjectDN - - TemplateContent - - NewTemplateContent - - SourcePort - - PasswordLastSet - - PrivilegeList - - DeviceDescription - - TargetServerName - - NewTargetUserName - - OperationType - - DestPort - - ServiceStartType - - OldTargetUserName - - UserPrincipalName - - Accesses - - DnsHostName - - DisableIntegrityChecks - - AuditSourceName - - Workstation - - DestAddress - - PreAuthType - - SecurityPackageName - - SubjectLogonId - - NewUacValue - - EnabledPrivilegeList - - RelativeTargetName - - CertSerialNumber - - SidHistory - - TargetLogonId - - KernelDebug - - CallerProcessName - - ProcessName - - Properties - - UserAccountControl - - RegistryValue - - SecurityID - - ServiceFileName - - SecurityDescriptor - - ServiceName - - ShareName - - NewValue - - Source - - Status - - SubjectDomainName - - SubjectUserName - - SubjectUserSid - - SourceAddr - - SourceAddress - - TargetName - - ServicePrincipalNames - - TargetDomainName - - TargetSid - - TargetUserName - - ObjectServer - - TargetUserSid - - TicketEncryptionType - - TicketOptions - - WorkstationName - - TransmittedServices - - AuthenticationAlgorithm - - LayerRTID - - BSSID - - BSSType - - CipherAlgorithm - - ConnectionId - - ConnectionMode - - InterfaceDescription - - InterfaceGuid - - OnexEnabled - - PHYType - - ProfileName - - SSID - - Domain - - ServiceType - - SourceName - - StartType - - UserID - - ParentProcessName - - ExceptionCode - - Service \ No newline at end of file + ParentImage: regex + AccessMask: regex + AccountName: regex + AllowedToDelegateTo: regex + AttributeLDAPDisplayName: regex + AuditPolicyChanges: regex + AuthenticationPackageName: regex + CallingProcessName: regex + Channel: regex + ComputerName: regex + EventType: regex + FailureReason: regex + FileName: regex + GrantedAccess: regex + Hashes: regex + HiveName: regex + IpAddress: regex + IpPort: regex + KeyLength: regex + LogonProcessName: regex + LogonType: regex + LinkName: regex + ProcessId: regex + PublishURLs: regex + ElevatedToken: regex + MemberName: regex + MemberSid: regex + NewProcessName: regex + ObjectClass: regex + ObjectName: regex + ObjectType: regex + ObjectValueName: regex + Path: regex + CommandLine: regex + OldUacValue: regex + CertIssuerName: regex + SubStatus: regex + DisplayName: regex + TaskContent: regex + ServiceSid: regex + CertThumbprint: regex + ClassName: regex + NotificationPackageName: regex + NewSd: regex + TestSigning: regex + TargetInfo: regex + ParentProcessId: regex + AccessList: regex + GroupMembership: regex + FilterName: regex + ChangeType: regex + LayerName: regex + ServiceAccount: regex + ClientProcessId: regex + AttributeValue: regex + SessionName: regex + TaskName: regex + ObjectDN: regex + TemplateContent: regex + NewTemplateContent: regex + SourcePort: regex + PasswordLastSet: regex + PrivilegeList: regex + DeviceDescription: regex + TargetServerName: regex + NewTargetUserName: regex + OperationType: regex + DestPort: regex + ServiceStartType: regex + OldTargetUserName: regex + UserPrincipalName: regex + Accesses: regex + DnsHostName: regex + DisableIntegrityChecks: regex + AuditSourceName: regex + Workstation: regex + DestAddress: regex + PreAuthType: regex + SecurityPackageName: regex + SubjectLogonId: regex + NewUacValue: regex + EnabledPrivilegeList: regex + RelativeTargetName: regex + CertSerialNumber: regex + SidHistory: regex + TargetLogonId: regex + KernelDebug: regex + CallerProcessName: regex + ProcessName: regex + Properties: regex + UserAccountControl: regex + RegistryValue: regex + SecurityID: regex + ServiceFileName: regex + SecurityDescriptor: regex + ServiceName: regex + ShareName: regex + NewValue: regex + Source: regex + Status: regex + SubjectDomainName: regex + SubjectUserName: regex + SubjectUserSid: regex + SourceAddr: regex + SourceAddress: regex + TargetName: regex + ServicePrincipalNames: regex + TargetDomainName: regex + TargetSid: regex + TargetUserName: regex + ObjectServer: regex + TargetUserSid: regex + TicketEncryptionType: regex + TicketOptions: regex + WorkstationName: regex + TransmittedServices: regex + AuthenticationAlgorithm: regex + LayerRTID: regex + BSSID: regex + BSSType: regex + CipherAlgorithm: regex + ConnectionId: regex + ConnectionMode: regex + InterfaceDescription: regex + InterfaceGuid: regex + OnexEnabled: regex + PHYType: regex + ProfileName: regex + SSID: regex + Domain: regex + ServiceType: regex + SourceName: regex + StartType: regex + UserID: regex + ParentProcessName: regex + ExceptionCode: regex + Service: regex + SamAccountName: regex + ImpersonationLevel: regex + PrimaryGroupId: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_sysmon.yml index ebfac1ec..a15909c9 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_sysmon.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_sysmon.yml @@ -11,50 +11,51 @@ field_mapping: Description: action_evtlog_description raw_log_fields: - - CommandLine - - Image - - ParentImage - - CallTrace - - Company - - CurrentDirectory - - DestinationHostname - - DestinationIp - - DestinationIsIpv6 - - DestinationPort - - DestinationPortName - - Hashes - - Initiated - - IntegrityLevel - - ParentCommandLine - - Product - - Protocol - - RuleName - - SourceHostname - - SourceIp - - SourceIsIpv6 - - SourcePort - - SourcePortName - - TargetFilename - - User - - Signed - - Signature - - SignatureStatus - - TargetObject - - Details - - QueryName - - QueryResults - - QueryStatus - - IsExecutable - - PipeName - - ImageLoaded - - ImagePath - - Imphash - - SourceImage - - StartModule - - TargetImage - - Device - - ProcessID - - FileVersion - - StartAddress - - StartFunction - - EventType \ No newline at end of file + CommandLine: regex + Image: regex + ParentImage: regex + CallTrace: regex + Company: regex + CurrentDirectory: regex + DestinationHostname: regex + DestinationIp: regex + DestinationIsIpv6: regex + DestinationPort: regex + DestinationPortName: regex + Hashes: regex + Initiated: regex + IntegrityLevel: regex + ParentCommandLine: regex + Product: regex + Protocol: regex + RuleName: regex + SourceHostname: regex + SourceIp: regex + SourceIsIpv6: regex + SourcePort: regex + SourcePortName: regex + TargetFilename: regex + User: regex + Signed: regex + Signature: regex + SignatureStatus: regex + TargetObject: regex + Details: regex + QueryName: regex + QueryResults: regex + QueryStatus: regex + IsExecutable: regex + PipeName: regex + ImageLoaded: regex + ImagePath: regex + Imphash: regex + SourceImage: regex + StartModule: regex + TargetImage: regex + Device: regex + ProcessID: regex + FileVersion: regex + StartAddress: regex + StartFunction: regex + EventType: regex + GrantedAccess: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_system.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_system.yml index d4bcb22a..07730124 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_system.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_system.yml @@ -10,14 +10,16 @@ field_mapping: ImagePath: actor_process_image_path raw_log_fields: - - AccountName - - ServiceName - - ServiceType - - StartType - - Origin - - HiveName - - Caption - - param1 - - param2 - - Channel - - DeviceName \ No newline at end of file + AccountName: regex + ServiceName: regex + ServiceType: regex + StartType: regex + Origin: regex + HiveName: regex + Caption: regex + param1: regex + param2: regex + Channel: regex + DeviceName: regex + Message: regex + ComputerName: regex \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml index 7efabc4e..6e798034 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml @@ -9,6 +9,7 @@ default_log_source: field_mapping: icmp.type: IcmpType + icmp.code: IcmpCode dst-port: - DstPort - DestinationPort @@ -26,14 +27,24 @@ field_mapping: - destination_ip - destinationIP - destinationaddress - User: userName + User: + - userName + - EventUserName CommandLine: Command Protocol: IPProtocol Application: - Application - application - SourceHostName: HostCount-source - DestinationHostname: HostCount-destination + SourceHostName: + - HostCount-source + - identityHostName + - sourceAssetName + - HostCount-src + DestinationHostname: + - HostCount-destination + - Recipient Host + - DestinationHostName + - HostCount-dst src-packets: - PacketRatio-src - src-packets @@ -41,4 +52,16 @@ field_mapping: - PacketRatio-dst - dst-packets src-bytes: src-bytes - dst-bytes: dst-bytes \ No newline at end of file + dst-bytes: dst-bytes + ExternalSeverity: External Severity + SourceMAC: + - SourceMAC + - MAC + DestinationMAC: DestinationMAC + SourceOS: + - SourceOS + - OS + DestinationOS: DestinationOS + TargetUserName: DestinationUserName + SourceUserName: SourceUserName + url_category: XForceCategoryByURL \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml b/uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml index cdeb8b82..14d7aefc 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml @@ -29,4 +29,5 @@ field_mapping: - DstPort - RemotePort Protocol: IPProtocol + application: Application Application: Application \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml index 2369e399..58393ac0 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml @@ -13,15 +13,24 @@ field_mapping: - URL - XForceCategoryByURL c-useragent: User Agent - cs-method: cs-method + cs-method: HTTP Method cs-bytes: Bytes Sent - cs-cookie-vars: cs-cookie-vars + #cs-cookie-vars: cs-cookie-vars c-uri-extension: URL - c-uri-query: URL - cs-cookie: cs-cookie - cs-host: cs-host - cs-referrer: URL Referrer - cs-version: cs-version - r-dns: r-dns - sc-status: sc-status - post-body: post-body \ No newline at end of file + c-uri-query: + - URL + - URL Path + #cs-cookie: cs-cookie + cs-host: + - UrlHost + - URL Host + cs-referrer: + - URL Referrer + - Referrer URL + cs-version: HTTP Version + r-dns: + - UrlHost + - URL Host + sc-status: HTTP Response Code + #post-body: post-body + url_category: XForceCategoryByURL \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_image_load.yml index 434114c0..bb1189f6 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_image_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_image_load.yml @@ -13,8 +13,12 @@ default_log_source: qideventcategory: Microsoft-Windows-Sysmon/Operational field_mapping: - Image: username - ImageLoaded: Process Path - SignatureStatus: Signature Status + Image: Process Path + ImageLoaded: + - Process Path + - LoadedImage + SignatureStatus: + - Signature Status + - SignatureStatus OriginalFileName: OriginalFileName Signed: Signed \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_termination.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_termination.yml new file mode 100644 index 00000000..563403a4 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_termination.yml @@ -0,0 +1,16 @@ +platform: Qradar +source: windows_process_termination + + +log_source: + devicetype: [12] + category: [8113] + +default_log_source: + devicetype: 12 + category: 8113 + +field_mapping: + Image: Process Path + ProcessId: ProcessId +# ProcessGuid: ProcessGuid \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml index a217b92c..7d01b97e 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml @@ -19,28 +19,44 @@ field_mapping: AuthenticationPackageName: AuthenticationPackageName CallingProcessName: CallingProcessName Channel: Channel - ComputerName: Machine Identifier + ComputerName: + - Machine Identifier + - Hostname EventType: EventType FailureReason: FailureReason FileName: Filename GrantedAccess: GrantedAccess Hashes: File Hash HiveName: HiveName - IpAddress: IpAddress - IpPort: IpPort + IpAddress: + - sourceip + - identityIP + IpPort: sourceport KeyLength: KeyLength LogonProcessName: LogonProcessName - LogonType: Logon Type + LogonType: + - Logon Type + - Login Type + - MSLogonType LinkName: LinkName MemberName: MemberName MemberSid: MemberSid NewProcessName: Process Name ObjectClass: ObjectClass - ObjectName: Object Name - ObjectType: Object Type + ObjectName: + - Object Name + - objectname + - MSFileObjectName + - ObjectName_Filename + - ObjectName + ObjectType: + - Object Type + - ObjectType ObjectValueName: ObjectValueName Path: Path - CommandLine: Command + CommandLine: + - Command + - Process Command Line OldUacValue: OldUacValue SubStatus: SubStatus DisplayName: DisplayName @@ -55,14 +71,18 @@ field_mapping: ClientProcessId: ClientProcessId ParentProcessId: ParentProcessId AccessList: AccessList - GroupMembership: GroupMembership + GroupMembership: + - GroupMembership + - GroupName FilterName: FilterName ChangeType: ChangeType LayerName: LayerName ServiceAccount: ServiceAccount AttributeValue: AttributeValue SessionName: SessionName - TaskName: TaskName + TaskName: + - TaskName + - Task Name ObjectDN: ObjectDN TemplateContent: TemplateContent NewTemplateContent: NewTemplateContent @@ -99,23 +119,35 @@ field_mapping: UserAccountControl: UserAccountControl RegistryValue: Target Object SecurityID: SecurityID - ServiceFileName: Service Filename + ServiceFileName: + - Service Filename + - ServiceFileName SecurityDescriptor: SecurityDescriptor ServiceName: Service Name - ShareName: Share Name + ShareName: + - Share Name + - ShareName NewValue: NewValue Source: Source Status: Status + SubjectAccountName: + - Subject Account Name + - SubjectAccountName SubjectDomainName: SubjectDomainName SubjectUserName: Target Username SubjectUserSid: SubjectUserSid SourceAddr: sourceip - SourceAddress: sourceip + SourceAddress: + - sourceip + - sourceaddress + TargetFilename: File Directory TargetName: Target Username ServicePrincipalNames: ServicePrincipalNames TargetDomainName: TargetDomainName TargetSid: TargetSid - TargetUserName: Target Username + TargetUserName: + - Target Username + - Target User Name ObjectServer: ObjectServer TargetUserSid: TargetUserSid TicketEncryptionType: TicketEncryptionType @@ -141,4 +173,5 @@ field_mapping: StartType: StartType UserID: UserID ParentProcessName: Parent Process Name - Service: Service \ No newline at end of file + Service: Service + hasIdentity: hasIdentity \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/sigma/aws_cloudtrail.yml b/uncoder-core/app/translator/mappings/platforms/sigma/aws_cloudtrail.yml index da509b89..7716ea12 100644 --- a/uncoder-core/app/translator/mappings/platforms/sigma/aws_cloudtrail.yml +++ b/uncoder-core/app/translator/mappings/platforms/sigma/aws_cloudtrail.yml @@ -13,7 +13,7 @@ default_log_source: field_mapping: eventSource: eventSource eventName: eventName - AdditionalEventData: additionalEventData.MFAUsed + AdditionalEventData.MFAUsed: additionalEventData.MFAUsed errorCode: errorCode errorMessage: errorMessage eventType: eventType diff --git a/uncoder-core/app/translator/mappings/platforms/sigma/azure_azureactivity.yml b/uncoder-core/app/translator/mappings/platforms/sigma/azure_azureactivity.yml index c8d090a5..7a17a916 100644 --- a/uncoder-core/app/translator/mappings/platforms/sigma/azure_azureactivity.yml +++ b/uncoder-core/app/translator/mappings/platforms/sigma/azure_azureactivity.yml @@ -4,7 +4,7 @@ source: azure_azureactivity log_source: product: [azure] - service: [azureactivity] + service: [azureactivity, activitylogs] default_log_source: product: azure diff --git a/uncoder-core/app/translator/mappings/platforms/sigma/azure_azuread.yml b/uncoder-core/app/translator/mappings/platforms/sigma/azure_azuread.yml index 54594bb0..d46b9688 100644 --- a/uncoder-core/app/translator/mappings/platforms/sigma/azure_azuread.yml +++ b/uncoder-core/app/translator/mappings/platforms/sigma/azure_azuread.yml @@ -4,7 +4,7 @@ source: azure_azuread log_source: product: [azure] - service: [azuread] + service: [azuread, auditlogs] default_log_source: product: azure diff --git a/uncoder-core/app/translator/mappings/platforms/sigma/azure_m365.yml b/uncoder-core/app/translator/mappings/platforms/sigma/azure_m365.yml index 7d2d1c46..b9877a5b 100644 --- a/uncoder-core/app/translator/mappings/platforms/sigma/azure_m365.yml +++ b/uncoder-core/app/translator/mappings/platforms/sigma/azure_m365.yml @@ -4,7 +4,7 @@ source: azure_m365 log_source: product: [azure] - service: [m365] + service: [m365, o365, office365] default_log_source: product: azure diff --git a/uncoder-core/app/translator/mappings/platforms/sigma/windows_pipe_created.yml b/uncoder-core/app/translator/mappings/platforms/sigma/windows_pipe_created.yml new file mode 100644 index 00000000..eb6cc32c --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/sigma/windows_pipe_created.yml @@ -0,0 +1,16 @@ +platform: Sigma +source: windows_pipe_created + + +log_source: + product: [windows] + category: [pipe_created] + +default_log_source: + product: windows + category: pipe_created + +field_mapping: + EventID: action_evtlog_event_id + PipeName: PipeName + Image: Image \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/sigma/windows_registry_event.yml b/uncoder-core/app/translator/mappings/platforms/sigma/windows_registry_event.yml index 867239aa..d44431b6 100644 --- a/uncoder-core/app/translator/mappings/platforms/sigma/windows_registry_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/sigma/windows_registry_event.yml @@ -3,7 +3,7 @@ source: windows_registry_event log_source: product: [windows] - category: [registry_event, registry_set] + category: [registry_event, registry_set, registry_delete, registry_add] default_log_source: product: windows diff --git a/uncoder-core/app/translator/platforms/__init__.py b/uncoder-core/app/translator/platforms/__init__.py index 33e4b1d2..f9c89733 100644 --- a/uncoder-core/app/translator/platforms/__init__.py +++ b/uncoder-core/app/translator/platforms/__init__.py @@ -1,17 +1,14 @@ -import importlib.util import os +from app.translator.tools.utils import execute_module from const import PLATFORMS_PATH -def init_platforms(): +def init_platforms() -> None: for platform in [f for f in os.listdir(PLATFORMS_PATH) if os.path.isdir(os.path.join(PLATFORMS_PATH, f))]: if not platform.startswith("__") and not platform.endswith("__"): # Platforms __init__.py execution - init_path = f"{PLATFORMS_PATH}/{platform}/__init__.py" - spec = importlib.util.spec_from_file_location("__init__", init_path) - init_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(init_module) + execute_module(f"{PLATFORMS_PATH}/{platform}/__init__.py") init_platforms() diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena.py b/uncoder-core/app/translator/platforms/athena/renders/athena.py index a717d94f..a62e5b00 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager from app.translator.platforms.athena.const import athena_details @@ -35,6 +36,9 @@ class AthenaQueryRender(SqlQueryRender): or_token = "OR" field_value_map = AthenaFieldValue(or_token=or_token) - query_pattern = "{prefix} WHERE {query} {functions}" comment_symbol = "--" is_single_line_comment = True + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"WHERE {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/base/aql/const.py b/uncoder-core/app/translator/platforms/base/aql/const.py index 063c6d78..1df7cdd1 100644 --- a/uncoder-core/app/translator/platforms/base/aql/const.py +++ b/uncoder-core/app/translator/platforms/base/aql/const.py @@ -1,5 +1,7 @@ UTF8_PAYLOAD_PATTERN = r"UTF8\(payload\)" NUM_VALUE_PATTERN = r"(?P\d+(?:\.\d+)*)" -SINGLE_QUOTES_VALUE_PATTERN = r"""'(?P(?:[:a-zA-Z\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{\}\[\]\s]|'')*)'""" +SINGLE_QUOTES_VALUE_PATTERN = ( + r"""'(?P(?:[:a-zA-Zа-яА-Я\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{\}\[\]\s]|'')*)'""" # noqa: RUF001 +) TABLE_PATTERN = r"\s+FROM\s+[a-zA-Z.\-*]+" TABLE_GROUP_PATTERN = r"\s+FROM\s+(?P[a-zA-Z.\-*]+)" diff --git a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py index 6792d900..05826d08 100644 --- a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -127,7 +128,6 @@ class AQLQueryRender(PlatformQueryRender): not_token = "NOT" field_value_map = AQLFieldValue(or_token=or_token) - query_pattern = "{prefix} AND {query} {functions}" def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) @@ -136,3 +136,7 @@ def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions def wrap_with_comment(self, value: str) -> str: return f"/* {value} */" + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"AND {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py index a5f0abdf..111ffd7d 100644 --- a/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/aql/str_value_manager.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy from typing import ClassVar diff --git a/uncoder-core/app/translator/platforms/base/aql/tokenizer.py b/uncoder-core/app/translator/platforms/base/aql/tokenizer.py index d2bfdfb7..54a797eb 100644 --- a/uncoder-core/app/translator/platforms/base/aql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/aql/tokenizer.py @@ -15,6 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ + import re from typing import ClassVar, Optional, Union diff --git a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py index 70760930..b5994499 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -106,8 +107,6 @@ class LuceneQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - query_pattern = "{query} {functions}" - comment_symbol = "//" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py b/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py index 0b5f3b8f..9eb8e6bc 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/base/lucene/str_value_manager.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import ClassVar from app.translator.core.str_value_manager import ( diff --git a/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py b/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py index 45fed5e4..eb54b7ea 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py @@ -15,6 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ + import re from typing import ClassVar, Optional, Union diff --git a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py index 995adf54..b2c12068 100644 --- a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -78,7 +79,6 @@ class SplQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - query_pattern = "{prefix} {query} {functions}" comment_symbol = "```" def wrap_with_comment(self, value: str) -> str: diff --git a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py index ebcb21af..43904a1e 100644 --- a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -76,10 +77,13 @@ class SqlQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - query_pattern = "{prefix} WHERE {query} {functions}" comment_symbol = "--" is_single_line_comment = True def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) if str(log_source_signature) else "eventlog" return f"SELECT * FROM {table}" + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"WHERE {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py index 7e511344..8c0e8431 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py @@ -16,7 +16,6 @@ ----------------------------------------------------------------- """ - from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import PlatformQueryParser diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py index 63f75608..4101b825 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -109,6 +110,5 @@ class ChronicleQueryRender(PlatformQueryRender): not_token = "not" field_value_map = ChronicleFieldValue(or_token=or_token) - query_pattern = "{query} {functions}" comment_symbol = "//" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py index 85b9635e..80130636 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py @@ -15,6 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ + from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager from app.translator.platforms.base.spl.parsers.spl import SplQueryParser diff --git a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py index 17ae1a15..8c6630e9 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender @@ -31,14 +32,13 @@ class CrowdStrikeFieldValue(SplFieldValue): @render_manager.register class CrowdStrikeQueryRender(SplQueryRender): details: PlatformDetails = crowdstrike_query_details - query_pattern = "{prefix} {query} {functions}" mappings: CrowdstrikeMappings = crowdstrike_mappings - platform_functions: CrowdStrikeFunctions = crowd_strike_functions + platform_functions: CrowdStrikeFunctions = None or_token = "OR" field_value_map = CrowdStrikeFieldValue(or_token=or_token) comment_symbol = "`" - def __init__(self): - super().__init__() - self.platform_functions.manager.post_init_configure(self) + def init_platform_functions(self) -> None: + self.platform_functions = crowd_strike_functions + self.platform_functions.platform_query_render = self diff --git a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py index 193365a2..8f7aed96 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py @@ -9,13 +9,13 @@ class ESQLEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { ValueType.value: [ EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), - EscapeDetails(pattern=r'"', escape_symbols=r"\"") + EscapeDetails(pattern=r'"', escape_symbols=r"\""), ], ValueType.regex_value: [ EscapeDetails(pattern=r'"', escape_symbols=r"\""), EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), - EscapeDetails(pattern=r'([_!@#$%^&*=+()\[\]{}|;:\'",.<>?/`~\-\s\\])', escape_symbols=r"\\\\\1") - ] + EscapeDetails(pattern=r'([_!@#$%^&*=+()\[\]{}|;:\'",.<>?/`~\-\s\\])', escape_symbols=r"\\\\\1"), + ], } diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py index 3e6a7823..dba7807a 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py @@ -16,7 +16,6 @@ ----------------------------------------------------------------- """ - from app.translator.core.mixins.rule import JsonRuleMixin from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py index 4e7face5..09fad79b 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -51,7 +51,6 @@ class ElasticSearchRuleRender(ElasticSearchQueryRender): not_token = "NOT" field_value_map = ElasticSearchRuleFieldValue(or_token=or_token) - query_pattern = "{prefix} {query} {functions}" def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): @@ -85,13 +84,14 @@ def finalize_query( query: str, functions: str, meta_info: Optional[MetaInfoContainer] = None, - source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 + source_mapping: Optional[SourceMapping] = None, not_supported_functions: Optional[list] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: query = super().finalize_query(prefix=prefix, query=query, functions=functions) rule = copy.deepcopy(ELASTICSEARCH_DETECTION_RULE) + index = source_mapping.log_source_signature.default_source.get("index") if source_mapping else None rule.update( { "query": query, @@ -105,6 +105,7 @@ def finalize_query( "tags": meta_info.tags, "threat": self.__create_mitre_threat(meta_info.mitre_attack), "false_positives": meta_info.false_positives, + "index": [index] if index else [], } ) rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py index ba1bb93b..104b8ecc 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Optional from app.translator.core.custom_types.meta_info import SeverityType @@ -49,7 +50,6 @@ class ElastAlertRuleRender(ElasticSearchQueryRender): not_token = "NOT" field_value_map = ElasticAlertRuleFieldValue(or_token=or_token) - query_pattern = "{prefix} {query} {functions}" def finalize_query( self, diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index abac539f..095ad61b 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -17,6 +17,7 @@ ----------------------------------------------------------------- """ from typing import Union + from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import LogSourceSignature @@ -37,10 +38,10 @@ def _make_case_insensitive(value: str) -> str: container: list[str] = [] for v in value: if v.isalpha(): - container.append(f'[{v.upper()}{v.lower()}]') + container.append(f"[{v.upper()}{v.lower()}]") else: container.append(v) - return ''.join(container) + return "".join(container) @staticmethod def _wrap_str_value(value: str) -> str: @@ -71,23 +72,25 @@ def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" - if field.endswith('.text'): + if field.endswith(".text"): return self.regex_modifier(field=field, value=value) return f'{field} like "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}*"' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if field.endswith('.text'): + if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f'ends_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})' + return f"ends_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})" def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: - if field.endswith('.text'): + if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return f'starts_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})' + return ( + f"starts_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})" + ) def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py index 31216239..c3b6a46a 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy import json from typing import Optional diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py index 551ac2c6..9a013dcf 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy import json from typing import Optional diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index bef9392b..65ca0b07 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -18,6 +18,7 @@ from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType @@ -192,7 +193,6 @@ class FortiSiemRuleRender(PlatformQueryRender): not_token = None group_token = "(%s)" - query_pattern = "{prefix} {query}" field_value_map = FortiSiemFieldValue(or_token=or_token) @@ -273,6 +273,8 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue source_mapping=source_mapping, fields=mapped_fields_set, ) + if return_only_first_query_ctx_var.get() is True: + return finalized_query queries_map[source_mapping.source_id] = finalized_query return self.finalize(queries_map) @@ -301,7 +303,7 @@ def finalize_query( self, prefix: str, query: str, - functions: str, # noqa: ARG002 + functions: str, meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, @@ -309,7 +311,7 @@ def finalize_query( *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: - query = self.query_pattern.format(prefix=prefix, query=query).strip() + query = super().finalize_query(prefix=prefix, query=query, functions=functions) rule = FORTI_SIEM_RULE.replace("", self.generate_rule_header(meta_info)) title = meta_info.title or _AUTOGENERATED_TEMPLATE rule = rule.replace("", self.generate_rule_name(title)) diff --git a/uncoder-core/app/translator/platforms/forti_siem/str_value_manager.py b/uncoder-core/app/translator/platforms/forti_siem/str_value_manager.py index 60d7198a..1c6b34ed 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/forti_siem/str_value_manager.py @@ -2,10 +2,13 @@ Uncoder IO Community Edition License ----------------------------------------------------------------- Copyright (c) 2024 SOC Prime, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py index 1dc54e94..0348bfb0 100644 --- a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py +++ b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender @@ -35,4 +36,7 @@ class HuntersQueryRender(SqlQueryRender): or_token = "OR" field_value_map = HuntersFieldValue(or_token=or_token) - query_pattern = "{prefix} WHERE {query} {functions}" + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"WHERE {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 9be24b73..624fa3d7 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -16,9 +16,11 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.core import StrictPlatformException @@ -203,13 +205,16 @@ class LogRhythmAxonQueryRender(PlatformQueryRender): not_token = "NOT" field_value_map = LogRhythmAxonFieldValue(or_token=or_token) - query_pattern = "{prefix} AND {query}" mappings: LogRhythmAxonMappings = logrhythm_axon_mappings comment_symbol = "//" is_single_line_comment = True is_strict_mapping = True + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"AND {query}" if query else "" + def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 return str(log_source_signature) @@ -262,6 +267,8 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue meta_info=query_container.meta_info, source_mapping=source_mapping, ) + if return_only_first_query_ctx_var.get() is True: + return finalized_query queries_map[source_mapping.source_id] = finalized_query return self.finalize(queries_map) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py index 7a250041..20514140 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy import json from typing import Optional diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py index fd9ede79..e1015ff2 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py @@ -16,7 +16,6 @@ ----------------------------------------------------------------- """ - from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py index f9a18c01..a9cbd603 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py @@ -16,7 +16,6 @@ ----------------------------------------------------------------- """ - from app.translator.core.mixins.rule import JsonRuleMixin from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py index a4e529ed..9cb7cf05 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -95,18 +96,17 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: class LogScaleQueryRender(PlatformQueryRender): details: PlatformDetails = logscale_query_details mappings: LogScaleMappings = logscale_mappings - platform_functions: LogScaleFunctions = log_scale_functions + platform_functions: LogScaleFunctions = None or_token = "or" and_token = "" not_token = "not" field_value_map = LogScaleFieldValue(or_token=or_token) - query_pattern = "{prefix} {query} {functions}" - def __init__(self): - super().__init__() - self.platform_functions.manager.post_init_configure(self) + def init_platform_functions(self) -> None: + self.platform_functions = log_scale_functions + self.platform_functions.platform_query_render = self def wrap_with_comment(self, value: str) -> str: return f"/* {value} */" @@ -122,10 +122,7 @@ def finalize_query( *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: - if prefix: - query = self.query_pattern.format(prefix=prefix, query=query, functions=functions) - else: - query = f"{query} {functions.lstrip()}" + query = super().finalize_query(prefix=prefix, query=query, functions=functions) query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) if not_supported_functions: rendered_not_supported = self.render_not_supported_functions(not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py index 24e9142f..4b3af0fb 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy import json from typing import Optional diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index 746c5cb0..507c8c17 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -16,7 +16,6 @@ ----------------------------------------------------------------- """ - from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py index c0615b57..9cf400e2 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py @@ -16,7 +16,6 @@ ----------------------------------------------------------------- """ - from app.translator.core.mixins.rule import JsonRuleMixin from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py index 59a8fe43..7b7a3779 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py @@ -36,8 +36,12 @@ class MicrosoftDefenderFieldValue(MicrosoftSentinelFieldValue): class MicrosoftDefenderQueryRender(MicrosoftSentinelQueryRender): mappings: MicrosoftDefenderMappings = microsoft_defender_mappings details: PlatformDetails = microsoft_defender_details - platform_functions: MicrosoftFunctions = microsoft_defender_functions + platform_functions: MicrosoftFunctions = None or_token = "or" field_value_map = MicrosoftDefenderFieldValue(or_token=or_token) is_strict_mapping = True + + def init_platform_functions(self) -> None: + self.platform_functions = microsoft_defender_functions + self.platform_functions.platform_query_render = self diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py index 372cb58d..621decb1 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import ClassVar from app.translator.core.models.platform_details import PlatformDetails diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py index cb32443a..3153f8d4 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -121,22 +122,25 @@ def is_not_none(self, field: str, value: Union[str, int]) -> str: # noqa: ARG00 @render_manager.register class MicrosoftSentinelQueryRender(PlatformQueryRender): details: PlatformDetails = microsoft_sentinel_query_details - platform_functions: MicrosoftFunctions = microsoft_sentinel_functions + platform_functions: MicrosoftFunctions = None or_token = "or" and_token = "and" not_token = "not" field_value_map = MicrosoftSentinelFieldValue(or_token=or_token) - query_pattern = "{prefix} | where {query}{functions}" mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings comment_symbol = "//" is_single_line_comment = True - def __init__(self): - super().__init__() - self.platform_functions.manager.post_init_configure(self) + def init_platform_functions(self) -> None: + self.platform_functions = microsoft_sentinel_functions + self.platform_functions.platform_query_render = self def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 return str(log_source_signature) + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"| where {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index 8a7089c5..e2fdb81f 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy import json from typing import Optional diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py index 23808279..1d2145a7 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py index 557f911e..3f68e6c6 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy import json from typing import Optional, Union @@ -49,7 +50,6 @@ class OpenSearchRuleRender(OpenSearchQueryRender): not_token = "NOT" field_value_map = OpenSearchRuleFieldValue(or_token=or_token) - query_pattern = "{prefix} {query} {functions}" def __init__(self): super().__init__() diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index b0d1e0f7..72a2737b 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -16,7 +16,8 @@ limitations under the License. ----------------------------------------------------------------- """ -from typing import Optional, Union + +from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType @@ -136,24 +137,41 @@ class CortexXQLQueryRender(PlatformQueryRender): details: PlatformDetails = cortex_xql_query_details mappings: CortexXQLMappings = cortex_xql_mappings is_strict_mapping = True - raw_log_field_pattern = ( - '| alter {field} = regextract(to_json_string(action_evtlog_data_fields)->{field}{{}}, "\\"(.*)\\"")' - ) - platform_functions: CortexXQLFunctions = cortex_xql_functions + raw_log_field_pattern_map: ClassVar[dict[str, str]] = { + "regex": '| alter {field} = regextract(to_json_string(action_evtlog_data_fields)->{field}{{}}, "\\"(.*)\\"")', + "object": '| alter {field_name} = json_extract_scalar({field_object} , "$.{field_path}")', + "list": '| alter {field_name} = arraystring(json_extract_array({field_object} , "$.{field_path}")," ")', + } + platform_functions: CortexXQLFunctions = None or_token = "or" and_token = "and" not_token = "not" field_value_map = CortexXQLFieldValue(or_token=or_token) - query_pattern = "{prefix} | filter {query} {functions}" comment_symbol = "//" is_single_line_comment = False - def __init__(self): - super().__init__() - self.platform_functions.manager.post_init_configure(self) + def init_platform_functions(self) -> None: + self.platform_functions = cortex_xql_functions + self.platform_functions.platform_query_render = self + + def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: + raw_log_field_pattern = self.raw_log_field_pattern_map.get(field_type) + if raw_log_field_pattern is None: + return + if field_type == "regex": + field = field.replace(".", r"\.") + return raw_log_field_pattern.format(field=field) + if field_type in ("object", "list") and "." in field: + field_object, field_path = field.split(".", 1) + field_name = field.replace(".", "_") + return raw_log_field_pattern.format(field_name=field_name, field_object=field_object, field_path=field_path) def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, functions_prefix: str = "") -> str: functions_prefix = f"{functions_prefix} | " if functions_prefix else "" return f"{functions_prefix}{log_source_signature}" + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"| filter {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/palo_alto/str_value_manager.py b/uncoder-core/app/translator/platforms/palo_alto/str_value_manager.py index 7a454d13..e547f223 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/palo_alto/str_value_manager.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy from app.translator.core.custom_types.values import ValueType diff --git a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py index e7c92b76..0f06fb40 100644 --- a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager from app.translator.platforms.base.aql.renders.aql import AQLFieldValue, AQLQueryRender diff --git a/uncoder-core/app/translator/platforms/sigma/escape_manager.py b/uncoder-core/app/translator/platforms/sigma/escape_manager.py index b656c4ad..c0efb332 100644 --- a/uncoder-core/app/translator/platforms/sigma/escape_manager.py +++ b/uncoder-core/app/translator/platforms/sigma/escape_manager.py @@ -7,7 +7,7 @@ class SigmaEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { - ValueType.value: [EscapeDetails(pattern=r'([*?\\])', escape_symbols=r"\\\1")], + ValueType.value: [EscapeDetails(pattern=r"([*?\\])", escape_symbols=r"\\\1")] } diff --git a/uncoder-core/app/translator/platforms/sigma/models/compiler.py b/uncoder-core/app/translator/platforms/sigma/models/compiler.py index 5969d06c..2c0b6472 100644 --- a/uncoder-core/app/translator/platforms/sigma/models/compiler.py +++ b/uncoder-core/app/translator/platforms/sigma/models/compiler.py @@ -15,6 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ + from typing import Union from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType diff --git a/uncoder-core/app/translator/platforms/sigma/models/modifiers.py b/uncoder-core/app/translator/platforms/sigma/models/modifiers.py index 7ae75726..446eb310 100644 --- a/uncoder-core/app/translator/platforms/sigma/models/modifiers.py +++ b/uncoder-core/app/translator/platforms/sigma/models/modifiers.py @@ -26,7 +26,7 @@ def map_modifier(self, modifier: str) -> Identifier: return Identifier(token_type=self.modifier_map.get(modifier, modifier)) def modifier_all(self, field_name: str, modifier: str, values: Union[str, list[str]]) -> Union[tuple, list]: - if (isinstance(values, list) and len(values) == 1) or isinstance(values, str): + if (isinstance(values, list) and len(values) == 1) or isinstance(values, (str, int)): operator = self.map_modifier(modifier=modifier) values = self.convert_values_to_str_values(values, modifier) return (FieldValue(source_name=field_name, operator=operator, value=values),) @@ -80,8 +80,7 @@ def apply_modifier(self, field_name: str, modifier: list, values: Union[int, str @staticmethod def convert_values_to_str_values( - values: Union[int, str, list[Union[int, str]]], - operator: str + values: Union[int, str, list[Union[int, str]]], operator: str ) -> Union[StrValue, list[StrValue]]: if not isinstance(values, list): values = [values] diff --git a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py index c5f1293b..9f2fd7ab 100644 --- a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py @@ -17,14 +17,13 @@ ----------------------------------------------------------------- """ - from typing import Union from app.translator.core.exceptions.core import SigmaRuleValidationException from app.translator.core.mixins.rule import YamlRuleMixin -from app.translator.core.models.field import FieldValue, Field -from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer, RawQueryContainer +from app.translator.core.models.field import Field, FieldValue from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import QueryParser from app.translator.core.tokenizer import QueryTokenizer from app.translator.managers import parser_manager @@ -50,12 +49,12 @@ def __parse_false_positives(false_positives: Union[str, list[str], None]) -> lis return false_positives def _get_meta_info( - self, - rule: dict, - source_mapping_ids: list[str], - parsed_logsources: dict, - fields_tokens: list[Field], - sigma_fields_tokens: Union[list[Field], None] = None + self, + rule: dict, + source_mapping_ids: list[str], + parsed_logsources: dict, + fields_tokens: list[Field], + sigma_fields_tokens: Union[list[Field], None] = None, ) -> MetaInfoContainer: return MetaInfoContainer( title=rule.get("title"), @@ -73,7 +72,7 @@ def _get_meta_info( tags=sorted(set(rule.get("tags", []))), false_positives=self.__parse_false_positives(rule.get("falsepositives")), source_mapping_ids=source_mapping_ids, - parsed_logsources=parsed_logsources + parsed_logsources=parsed_logsources, ) def __validate_rule(self, rule: dict): @@ -97,10 +96,11 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain source_mappings = self.mappings.get_suitable_source_mappings(field_names=field_names, **log_sources) QueryTokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping) sigma_fields_tokens = None - if sigma_fields := sigma_rule.get('fields'): + if sigma_fields := sigma_rule.get("fields"): sigma_fields_tokens = [Field(source_name=field) for field in sigma_fields] - QueryTokenizer.set_field_tokens_generic_names_map(sigma_fields_tokens, source_mappings, - self.mappings.default_mapping) + QueryTokenizer.set_field_tokens_generic_names_map( + sigma_fields_tokens, source_mappings, self.mappings.default_mapping + ) return TokenizedQueryContainer( tokens=tokens, meta_info=self._get_meta_info( @@ -108,6 +108,6 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain source_mapping_ids=[source_mapping.source_id for source_mapping in source_mappings], sigma_fields_tokens=sigma_fields_tokens, parsed_logsources=log_sources, - fields_tokens=field_tokens - ) + fields_tokens=field_tokens, + ), ) diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index b0e49ee1..dc33a507 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -25,8 +25,8 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.query_container import TokenizedQueryContainer, RawQueryContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.render import QueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager diff --git a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py index c73115e7..7b1ccee1 100644 --- a/uncoder-core/app/translator/platforms/sigma/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/sigma/str_value_manager.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from app.translator.core.str_value_manager import ( ReAnySymbol, ReCaretSymbol, @@ -57,7 +58,7 @@ "}": ReRightCurlyBracket, "|": ReOrOperator, ",": ReCommaSymbol, - "-": ReHyphenSymbol + "-": ReHyphenSymbol, } diff --git a/uncoder-core/app/translator/platforms/sigma/tokenizer.py b/uncoder-core/app/translator/platforms/sigma/tokenizer.py index bb1736dd..0893588f 100644 --- a/uncoder-core/app/translator/platforms/sigma/tokenizer.py +++ b/uncoder-core/app/translator/platforms/sigma/tokenizer.py @@ -28,6 +28,7 @@ class Selection: token_type = "selection" + def __init__(self, name): self.name = name @@ -142,10 +143,12 @@ def get_missed_parentheses(tokens: list[Union[Selection, Identifier]]) -> list[i missed_indices.append(index + 1) return missed_indices - def __add_parentheses_after_and_not(self, tokens: list[Union[Selection, Identifier]]) -> list[Union[Selection, Identifier]]: + def __add_parentheses_after_and_not( + self, tokens: list[Union[Selection, Identifier]] + ) -> list[Union[Selection, Identifier]]: indices = self.get_missed_parentheses(tokens=tokens) for index in reversed(indices): - tokens.insert(index+1, Identifier(token_type=GroupType.R_PAREN)) + tokens.insert(index + 1, Identifier(token_type=GroupType.R_PAREN)) tokens.insert(index, Identifier(token_type=GroupType.L_PAREN)) return tokens diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py index 15a131b0..f9404cac 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender @@ -36,8 +37,8 @@ class SplunkQueryRender(SplQueryRender): field_value_map = SplunkFieldValue(or_token=or_token) mappings: SplunkMappings = splunk_mappings - platform_functions: SplunkFunctions = splunk_functions + platform_functions: SplunkFunctions = None - def __init__(self): - super().__init__() - self.platform_functions.manager.post_init_configure(self) + def init_platform_functions(self) -> None: + self.platform_functions = splunk_functions + self.platform_functions.platform_query_render = self diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py index 19acb808..ef0d097d 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Optional from app.translator.core.custom_types.meta_info import SeverityType diff --git a/uncoder-core/app/translator/tools/utils.py b/uncoder-core/app/translator/tools/utils.py index f48049ad..1aba4ebf 100644 --- a/uncoder-core/app/translator/tools/utils.py +++ b/uncoder-core/app/translator/tools/utils.py @@ -1,7 +1,16 @@ +import importlib.util import re +from contextlib import suppress from typing import Optional, Union +def execute_module(path: str) -> None: + with suppress(FileNotFoundError): + spec = importlib.util.spec_from_file_location("__init__", path) + init_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(init_module) + + def get_match_group(match: re.Match, group_name: str) -> Optional[str]: try: return match.group(group_name) From 79611814c0d90e9dcd8be826f018da03c3a5dac2 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Thu, 20 Jun 2024 09:27:35 +0200 Subject: [PATCH 10/40] remove inheritance from sql --- .../translator/platforms/elasticsearch/renders/esql.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 095ad61b..401d0b1e 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -22,14 +22,14 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.managers import render_manager -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings from app.translator.platforms.elasticsearch.str_value_manager import ESQLStrValueManager, esql_str_value_manager -class ESQLFieldValue(SqlFieldValue): +class ESQLFieldValue(BaseQueryFieldValue): details: PlatformDetails = elasticsearch_esql_query_details str_value_manager: ESQLStrValueManager = esql_str_value_manager @@ -104,7 +104,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @render_manager.register -class ESQLQueryRender(SqlQueryRender): +class ESQLQueryRender(PlatformQueryRender): details: PlatformDetails = elasticsearch_esql_query_details mappings: ElasticSearchMappings = elasticsearch_mappings comment_symbol = "//" @@ -117,3 +117,7 @@ class ESQLQueryRender(SqlQueryRender): def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) if str(log_source_signature) else "*" return f"FROM {table} metadata _id, _version, _index |" + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"WHERE {query}" if query else "" From 065f8d2aa0ed973452af3de465841b67d5a2c6e3 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Thu, 20 Jun 2024 13:42:45 +0200 Subject: [PATCH 11/40] update esql escape manager --- .../platforms/elasticsearch/escape_manager.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py index 8f7aed96..ec45867c 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py @@ -8,13 +8,14 @@ class ESQLEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { ValueType.value: [ - EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), - EscapeDetails(pattern=r'"', escape_symbols=r"\""), + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\\\\\"), + EscapeDetails(pattern=r'"', escape_symbols=r"\\\""), + EscapeDetails(pattern=r"'", escape_symbols=r"\'"), ], ValueType.regex_value: [ - EscapeDetails(pattern=r'"', escape_symbols=r"\""), - EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\"), - EscapeDetails(pattern=r'([_!@#$%^&*=+()\[\]{}|;:\'",.<>?/`~\-\s\\])', escape_symbols=r"\\\\\1"), + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\\\\\"), + EscapeDetails(pattern=r'"', escape_symbols=r"\\\""), + EscapeDetails(pattern=r"'", escape_symbols=r"\'"), ], } From 3ee5aa50cb824872afb14a49f74504338b5ea3a5 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Thu, 20 Jun 2024 13:54:24 +0200 Subject: [PATCH 12/40] update esql escape manager From 51ac34f259a2abf589e9489d6323507d5d6c6130 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Thu, 20 Jun 2024 13:54:29 +0200 Subject: [PATCH 13/40] update typing + refactoring in esql rule --- uncoder-core/app/translator/core/render.py | 8 ++--- .../elasticsearch/renders/esql_rule.py | 33 +++++++++++-------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 055ce889..498cc660 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -266,7 +266,7 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) return "".join(result_values) - def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str: + def wrap_query_with_meta_info(self, meta_info: Optional[MetaInfoContainer], query: str) -> str: if meta_info and (meta_info.id or meta_info.title): meta_info_dict = { "name: ": meta_info.title, @@ -289,9 +289,9 @@ def finalize_query( prefix: str, query: str, functions: str, - meta_info: Optional[MetaInfoContainer] = None, - source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 - not_supported_functions: Optional[list] = None, + meta_info: Union[MetaInfoContainer, None] = None, + source_mapping: Union[SourceMapping, None] = None, # noqa: ARG002 + not_supported_functions: Union[list, None] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py index 8802456e..0e2659e7 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -18,7 +18,7 @@ """ import copy import json -from typing import Optional, Union +from typing import Union from app.translator.core.mapping import SourceMapping from app.translator.core.mitre import MitreConfig @@ -73,9 +73,9 @@ def finalize_query( prefix: str, query: str, functions: str, - meta_info: MetaInfoContainer, - source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 - not_supported_functions: Optional[list] = None, + meta_info: Union[MetaInfoContainer, None] = None, + source_mapping: Union[SourceMapping, None] = None, # noqa: ARG002 + not_supported_functions: Union[list, None] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -84,18 +84,23 @@ def finalize_query( rule.update( { "query": query, - "description": meta_info.description or rule["description"] or _AUTOGENERATED_TEMPLATE, - "name": meta_info.title or _AUTOGENERATED_TEMPLATE, - "rule_id": meta_info.id, - "author": [meta_info.author], - "severity": meta_info.severity, - "references": meta_info.references, - "license": meta_info.license, - "tags": meta_info.tags, - "threat": self.__create_mitre_threat(meta_info.mitre_attack), - "false_positives": meta_info.false_positives, + "description": meta_info.description if meta_info else rule["description"] or _AUTOGENERATED_TEMPLATE, + "name": meta_info.title if meta_info else _AUTOGENERATED_TEMPLATE, } ) + if meta_info: + rule.update( + { + "rule_id": meta_info.id, + "author": [meta_info.author], + "severity": meta_info.severity, + "references": meta_info.references, + "license": meta_info.license, + "tags": meta_info.tags, + "threat": self.__create_mitre_threat(meta_info.mitre_attack), + "false_positives": meta_info.false_positives, + } + ) rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) if not_supported_functions: rendered_not_supported = self.render_not_supported_functions(not_supported_functions) From 17d8d651aa558fd0b374a198ed8cb6816173b30e Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:18:56 +0200 Subject: [PATCH 14/40] update esql escape manager From 25e71de249f037889f5de5cc67f8b70c3ee3b567 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:19:00 +0200 Subject: [PATCH 15/40] update typing + refactoring in esql rule From ec659a7d1d012f1d8b523f927f0036d248e6f7b6 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:19:04 +0200 Subject: [PATCH 16/40] update typing --- uncoder-core/app/translator/core/render.py | 6 +++--- .../platforms/elasticsearch/renders/esql_rule.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 498cc660..1cf2d1d5 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -289,9 +289,9 @@ def finalize_query( prefix: str, query: str, functions: str, - meta_info: Union[MetaInfoContainer, None] = None, - source_mapping: Union[SourceMapping, None] = None, # noqa: ARG002 - not_supported_functions: Union[list, None] = None, + meta_info: Optional[MetaInfoContainer] = None, + source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 + not_supported_functions: Optional[list] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py index 0e2659e7..a118ba3c 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -18,7 +18,7 @@ """ import copy import json -from typing import Union +from typing import Optional, Union from app.translator.core.mapping import SourceMapping from app.translator.core.mitre import MitreConfig @@ -73,9 +73,9 @@ def finalize_query( prefix: str, query: str, functions: str, - meta_info: Union[MetaInfoContainer, None] = None, - source_mapping: Union[SourceMapping, None] = None, # noqa: ARG002 - not_supported_functions: Union[list, None] = None, + meta_info: Optional[MetaInfoContainer] = None, + source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 + not_supported_functions: Optional[list] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: From 6ed63fb5d3a3ab23bcac1c9f203359298bcc0c33 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 2 Jul 2024 09:55:30 +0200 Subject: [PATCH 17/40] merge prod --- uncoder-core/app/translator/core/const.py | 6 + .../app/translator/core/context_vars.py | 6 + .../core/custom_types/predefined_fields.py | 12 ++ .../app/translator/core/custom_types/time.py | 14 ++ .../app/translator/core/exceptions/core.py | 18 ++- .../app/translator/core/models/field.py | 46 ++++++- .../translator/core/models/query_container.py | 2 +- uncoder-core/app/translator/core/parser.py | 3 +- uncoder-core/app/translator/core/render.py | 130 ++++++++++++------ uncoder-core/app/translator/core/tokenizer.py | 18 ++- .../platforms/palo_alto_cortex/default.yml | 1 + .../platforms/palo_alto_cortex/dns.yml | 3 +- .../linux_process_creation.yml | 3 +- .../windows_registry_event.yml | 3 +- .../mappings/platforms/qradar/default.yml | 21 ++- .../mappings/platforms/qradar/dns.yml | 3 +- .../mappings/platforms/qradar/firewall.yml | 3 + .../qradar/linux_process_creation.yml | 7 +- .../mappings/platforms/qradar/proxy.yml | 1 + .../qradar/windows_process_creation.yml | 13 +- .../qradar/windows_process_termination.yml | 4 +- .../platforms/qradar/windows_security.yml | 28 +++- .../platforms/splunk/aws_cloudtrail.yml | 4 +- .../mappings/platforms/splunk/aws_eks.yml | 4 +- .../splunk/azure_AzureDiagnostics.yml | 4 +- .../splunk/azure_BehaviorAnalytics.yml | 4 +- .../azure_aadnoninteractiveusersigninlogs.yml | 4 +- .../platforms/splunk/azure_azureactivity.yml | 4 +- .../platforms/splunk/azure_azuread.yml | 4 +- .../platforms/splunk/azure_signinlogs.yml | 4 +- .../mappings/platforms/splunk/firewall.yml | 4 +- .../platforms/splunk/gcp_gcp.audit.yml | 2 +- .../mappings/platforms/splunk/gcp_pubsub.yml | 2 +- .../platforms/splunk/linux_auditd.yml | 4 +- .../mappings/platforms/splunk/okta_okta.yml | 4 +- .../platforms/splunk/windows_bits_client.yml | 4 +- .../platforms/splunk/windows_dns_query.yml | 4 +- .../platforms/splunk/windows_driver_load.yml | 4 +- .../platforms/splunk/windows_file_access.yml | 4 +- .../platforms/splunk/windows_file_change.yml | 4 +- .../platforms/splunk/windows_file_create.yml | 4 +- .../platforms/splunk/windows_file_delete.yml | 4 +- .../platforms/splunk/windows_file_event.yml | 4 +- .../platforms/splunk/windows_file_rename.yml | 4 +- .../platforms/splunk/windows_image_load.yml | 4 +- .../platforms/splunk/windows_ldap_debug.yml | 4 +- .../splunk/windows_network_connection.yml | 4 +- .../platforms/splunk/windows_ntlm.yml | 4 +- .../splunk/windows_registry_event.yml | 4 +- .../platforms/splunk/windows_sysmon.yml | 4 +- .../platforms/splunk/windows_wmi_event.yml | 4 +- .../platforms/athena/renders/athena.py | 6 +- .../platforms/base/aql/renders/aql.py | 6 +- .../platforms/base/lucene/renders/lucene.py | 4 +- .../platforms/base/spl/renders/spl.py | 4 +- .../platforms/base/sql/renders/sql.py | 4 +- .../platforms/chronicle/renders/chronicle.py | 6 +- .../chronicle/renders/chronicle_rule.py | 6 +- .../crowdstrike/renders/crowdstrike.py | 6 +- .../elasticsearch/renders/detection_rule.py | 7 +- .../elasticsearch/renders/elast_alert.py | 7 +- .../elasticsearch/renders/elasticsearch.py | 6 +- .../platforms/elasticsearch/renders/kibana.py | 7 +- .../elasticsearch/renders/xpack_watcher.py | 7 +- .../forti_siem/renders/forti_siem_rule.py | 16 +-- .../platforms/graylog/renders/graylog.py | 6 +- .../platforms/hunters/renders/hunters.py | 6 +- .../renders/logrhythm_axon_query.py | 27 ++-- .../renders/logrhythm_axon_rule.py | 11 +- .../platforms/logscale/renders/logscale.py | 13 +- .../logscale/renders/logscale_alert.py | 13 +- .../microsoft/renders/microsoft_defender.py | 6 +- .../microsoft/renders/microsoft_sentinel.py | 6 +- .../renders/microsoft_sentinel_rule.py | 11 +- .../opensearch/renders/opensearch.py | 6 +- .../opensearch/renders/opensearch_rule.py | 13 +- .../translator/platforms/palo_alto/const.py | 13 ++ .../palo_alto/renders/cortex_xsiam.py | 66 +++++++-- .../platforms/qradar/renders/qradar.py | 5 +- .../app/translator/platforms/sigma/mapping.py | 6 +- .../platforms/sigma/renders/sigma.py | 8 +- .../translator/platforms/splunk/mapping.py | 4 +- .../platforms/splunk/renders/splunk.py | 6 +- .../platforms/splunk/renders/splunk_alert.py | 11 +- 84 files changed, 497 insertions(+), 289 deletions(-) create mode 100644 uncoder-core/app/translator/core/const.py create mode 100644 uncoder-core/app/translator/core/custom_types/predefined_fields.py diff --git a/uncoder-core/app/translator/core/const.py b/uncoder-core/app/translator/core/const.py new file mode 100644 index 00000000..a8788ada --- /dev/null +++ b/uncoder-core/app/translator/core/const.py @@ -0,0 +1,6 @@ +from typing import Union + +from app.translator.core.models.field import Alias, Field, FieldValue, Keyword +from app.translator.core.models.identifier import Identifier + +TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias] diff --git a/uncoder-core/app/translator/core/context_vars.py b/uncoder-core/app/translator/core/context_vars.py index 591883d8..8ff6ccfb 100644 --- a/uncoder-core/app/translator/core/context_vars.py +++ b/uncoder-core/app/translator/core/context_vars.py @@ -1,4 +1,10 @@ from contextvars import ContextVar +from typing import Optional return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False) """Set to True to return only first query if rendered multiple options""" + +wrap_query_with_meta_info_ctx_var: ContextVar[bool] = ContextVar("wrap_query_with_meta_info_ctx_var", default=True) +"""Set to False not to wrap query with meta info commentary""" + +preset_log_source_str_ctx_var: ContextVar[Optional[str]] = ContextVar("preset_log_source_str_ctx_var", default=None) diff --git a/uncoder-core/app/translator/core/custom_types/predefined_fields.py b/uncoder-core/app/translator/core/custom_types/predefined_fields.py new file mode 100644 index 00000000..50cc0cb7 --- /dev/null +++ b/uncoder-core/app/translator/core/custom_types/predefined_fields.py @@ -0,0 +1,12 @@ +from app.translator.tools.custom_enum import CustomEnum + + +class IPLocationType(CustomEnum): + asn = "ip_loc_asn" + asn_org = "ip_loc_asn_org" + city = "ip_loc_city" + continent = "ip_loc_continent" + country = "ip_loc_country" + lat_lon = "ip_loc_lat_lon" + region = "ip_loc_region" + timezone = "ip_loc_timezone" diff --git a/uncoder-core/app/translator/core/custom_types/time.py b/uncoder-core/app/translator/core/custom_types/time.py index 1d5f15b8..4cdc71fe 100644 --- a/uncoder-core/app/translator/core/custom_types/time.py +++ b/uncoder-core/app/translator/core/custom_types/time.py @@ -7,3 +7,17 @@ class TimeFrameType(CustomEnum): days = "days" hours = "hours" minutes = "minutes" + + +class TimePartType(CustomEnum): + day = "day" + day_of_week = "day_of_week" + day_of_year = "day_of_year" + hour = "hour" + microsecond = "microsecond" + millisecond = "millisecond" + minute = "minute" + month = "month" + quarter = "quarter" + second = "second" + year = "year" diff --git a/uncoder-core/app/translator/core/exceptions/core.py b/uncoder-core/app/translator/core/exceptions/core.py index 68c66962..47810576 100644 --- a/uncoder-core/app/translator/core/exceptions/core.py +++ b/uncoder-core/app/translator/core/exceptions/core.py @@ -1,3 +1,6 @@ +from typing import Optional + + class NotImplementedException(BaseException): ... @@ -7,8 +10,19 @@ class BasePlatformException(BaseException): class StrictPlatformException(BasePlatformException): - def __init__(self, platform_name: str, field_name: str): - message = f"Platform {platform_name} has strict mapping. Source field {field_name} has no mapping." + field_name: str = None + + def __init__( + self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None + ): + message = ( + f"Platform {platform_name} has strict mapping. " + f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping." + f" Mapping file: {mapping}." + if mapping + else "" + ) + self.field_name = field_name super().__init__(message) diff --git a/uncoder-core/app/translator/core/models/field.py b/uncoder-core/app/translator/core/models/field.py index 10b661b0..d9facb77 100644 --- a/uncoder-core/app/translator/core/models/field.py +++ b/uncoder-core/app/translator/core/models/field.py @@ -37,6 +37,27 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma self.__generic_names_map = generic_names_map +class PredefinedField: + def __init__(self, name: str): + self.name = name + + +class FieldField: + def __init__( + self, + source_name_left: str, + operator: Identifier, + source_name_right: str, + is_alias_left: bool = False, + is_alias_right: bool = False, + ): + self.field_left = Field(source_name=source_name_left) if not is_alias_left else None + self.alias_left = Alias(name=source_name_left) if is_alias_left else None + self.operator = operator + self.field_right = Field(source_name=source_name_right) if not is_alias_right else None + self.alias_right = Alias(name=source_name_right) if is_alias_right else None + + class FieldValue: def __init__( self, @@ -44,11 +65,14 @@ def __init__( operator: Identifier, value: Union[int, str, StrValue, list, tuple], is_alias: bool = False, + is_predefined_field: bool = False, ): - self.field = Field(source_name=source_name) - self.alias = None - if is_alias: - self.alias = Alias(name=source_name) + # mapped by platform fields mapping + self.field = Field(source_name=source_name) if not (is_alias or is_predefined_field) else None + # not mapped + self.alias = Alias(name=source_name) if is_alias else None + # mapped by platform predefined fields mapping + self.predefined_field = PredefinedField(name=source_name) if is_predefined_field else None self.operator = operator self.values = [] @@ -60,6 +84,11 @@ def value(self) -> Union[int, str, StrValue, list[Union[int, str, StrValue]]]: return self.values[0] return self.values + @value.setter + def value(self, new_value: Union[int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: + self.values = [] + self.__add_value(new_value) + def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -> None: if value and isinstance(value, (list, tuple)): for v in value: @@ -75,10 +104,13 @@ def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) - self.values.append(value) def __repr__(self): - if self.field: - return f"{self.field.source_name} {self.operator.token_type} {self.values}" + if self.alias: + return f"{self.alias.name} {self.operator.token_type} {self.values}" + + if self.predefined_field: + return f"{self.predefined_field.name} {self.operator.token_type} {self.values}" - return f"{self.alias.name} {self.operator.token_type} {self.values}" + return f"{self.field.source_name} {self.operator.token_type} {self.values}" class Keyword: diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index dccfc180..0d90f237 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -3,11 +3,11 @@ from datetime import datetime from typing import Optional +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import DEFAULT_MAPPING_NAME from app.translator.core.models.field import Field from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.tokenizer import TOKEN_TYPE class MetaInfoContainer: diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 7cc10ec1..18b50739 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping @@ -28,7 +29,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer -from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer +from app.translator.core.tokenizer import QueryTokenizer class QueryParser(ABC): diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 1cf2d1d5..b4b1ccc5 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -16,13 +16,14 @@ limitations under the License. ----------------------------------------------------------------- """ - +import itertools from abc import ABC, abstractmethod from collections.abc import Callable from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.const import TOKEN_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -30,22 +31,21 @@ from app.translator.core.exceptions.parser import UnsupportedOperatorException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword, PredefinedField from app.translator.core.models.functions.base import Function, RenderedFunctions from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.str_value_manager import StrValue, StrValueManager -from app.translator.core.tokenizer import TOKEN_TYPE -class BaseQueryFieldValue(ABC): +class BaseFieldValueRender(ABC): details: PlatformDetails = None escape_manager: EscapeManager = None str_value_manager: StrValueManager = None def __init__(self, or_token: str): - self.field_value: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { + self.modifiers_map: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { OperatorType.EQ: self.equal_modifier, OperatorType.NOT_EQ: self.not_equal_modifier, OperatorType.LT: self.less_modifier, @@ -155,11 +155,20 @@ def apply_value(self, value: Union[str, int], value_type: str = ValueType.value) return self.escape_manager.escape(value, value_type) def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VALUE_TYPE) -> str: - if modifier_function := self.field_value.get(operator.token_type): + if modifier_function := self.modifiers_map.get(operator.token_type): return modifier_function(field, value) raise UnsupportedOperatorException(operator.token_type) +class BaseFieldFieldRender(ABC): + operators_map: ClassVar[dict[str, str]] = {} + + def apply_field_field(self, field_left: str, operator: Identifier, field_right: str) -> str: + if mapped_operator := self.operators_map.get(operator.token_type): + return f"{field_left} {mapped_operator} {field_right}" + raise UnsupportedOperatorException(operator.token_type) + + class QueryRender(ABC): comment_symbol: str = None details: PlatformDetails = None @@ -180,6 +189,13 @@ def render_not_supported_functions(self, not_supported_functions: list) -> str: not_supported_functions_str = "\n".join(line_template + func.lstrip() for func in not_supported_functions) return "\n\n" + self.wrap_with_comment(f"{self.unsupported_functions_text}\n{not_supported_functions_str}") + def wrap_with_not_supported_functions(self, query: str, not_supported_functions: Optional[list] = None) -> str: + if not_supported_functions and wrap_query_with_meta_info_ctx_var.get(): + rendered_not_supported = self.render_not_supported_functions(not_supported_functions) + return query + rendered_not_supported + + return query + def wrap_with_comment(self, value: str) -> str: return f"{self.comment_symbol} {value}" @@ -197,14 +213,17 @@ class PlatformQueryRender(QueryRender): not_token = "not" group_token = "(%s)" + query_parts_delimiter = " " - field_value_map = BaseQueryFieldValue(or_token=or_token) + field_field_render = BaseFieldFieldRender() + field_value_render = BaseFieldValueRender(or_token=or_token) - raw_log_field_pattern_map: ClassVar[dict[str, str]] = None + predefined_fields_map: ClassVar[dict[str, str]] = {} + raw_log_field_patterns_map: ClassVar[dict[str, str]] = {} def __init__(self): super().__init__() - self.operator_map = { + self.logical_operators_map = { LogicalOperatorType.AND: f" {self.and_token} ", LogicalOperatorType.OR: f" {self.or_token} ", LogicalOperatorType.NOT: f" {self.not_token} ", @@ -230,44 +249,67 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]: return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name] + def map_predefined_field(self, predefined_field: PredefinedField) -> str: + if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)): + if self.is_strict_mapping: + raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name) + + return predefined_field.name + + return mapped_predefined_field_name + def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): if token.alias: - field_name = token.alias.name + mapped_fields = [token.alias.name] + elif token.predefined_field: + mapped_fields = [self.map_predefined_field(token.predefined_field)] else: mapped_fields = self.map_field(token.field, source_mapping) - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value( - field=field, operator=token.operator, value=token.value - ) - for field in mapped_fields - ] - ) - - field_name = mapped_fields[0] - - return self.field_value_map.apply_field_value(field=field_name, operator=token.operator, value=token.value) - + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] + ) + return self.group_token % joined if len(mapped_fields) > 1 else joined + if isinstance(token, FieldField): + alias_left, field_left = token.alias_left, token.field_left + mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping) + alias_right, field_right = token.alias_right, token.field_right + mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping) + cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right)) + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_field_render.apply_field_field(pair[0], token.operator, pair[1]) + for pair in cross_paired_fields + ] + ) + return self.group_token % joined if len(cross_paired_fields) > 1 else joined if isinstance(token, Function): func_render = self.platform_functions.manager.get_in_query_render(token.name) return func_render.render(token, source_mapping) if isinstance(token, Keyword): - return self.field_value_map.apply_field_value(field="", operator=token.operator, value=token.value) + return self.field_value_render.apply_field_value(field="", operator=token.operator, value=token.value) if token.token_type in LogicalOperatorType: - return self.operator_map.get(token.token_type) + return self.logical_operators_map.get(token.token_type) return token.token_type def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str: result_values = [] + unmapped_fields = set() for token in tokens: - result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) + try: + result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) + except StrictPlatformException as err: + unmapped_fields.add(err.field_name) + if unmapped_fields: + raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields)) return "".join(result_values) - def wrap_query_with_meta_info(self, meta_info: Optional[MetaInfoContainer], query: str) -> str: - if meta_info and (meta_info.id or meta_info.title): + def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str: + if wrap_query_with_meta_info_ctx_var.get() and meta_info and (meta_info.id or meta_info.title): meta_info_dict = { "name: ": meta_info.title, "uuid: ": meta_info.id, @@ -284,6 +326,10 @@ def wrap_query_with_meta_info(self, meta_info: Optional[MetaInfoContainer], quer def _finalize_search_query(query: str) -> str: return query + def _join_query_parts(self, prefix: str, query: str, functions: str) -> str: + parts = filter(lambda s: bool(s), map(str.strip, [prefix, self._finalize_search_query(query), functions])) + return self.query_parts_delimiter.join(parts) + def finalize_query( self, prefix: str, @@ -295,13 +341,9 @@ def finalize_query( *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: - parts = filter(lambda s: bool(s), map(str.strip, [prefix, self._finalize_search_query(query), functions])) - query = " ".join(parts) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self._join_query_parts(prefix, query, functions) + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) @staticmethod def unique_queries(queries_map: dict[str, str]) -> dict[str, dict[str]]: @@ -332,13 +374,13 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp return source_mappings - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: return self.finalize_query( prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info ) def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: - if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type): + if raw_log_field_pattern := self.raw_log_field_patterns_map.get(field_type): return raw_log_field_pattern.format(field=field) def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[list]: @@ -352,7 +394,7 @@ def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)] def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str: - if self.raw_log_field_pattern_map is None: + if not self.raw_log_field_patterns_map: return "" defined_raw_log_fields = [] for field in fields: @@ -370,7 +412,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap defined_raw_log_fields.append(prefix) return "\n".join(defined_raw_log_fields) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} errors = [] source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -383,7 +425,7 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue defined_raw_log_fields = self.generate_raw_log_fields( fields=query_container.meta_info.query_fields, source_mapping=source_mapping ) - prefix += f"\n{defined_raw_log_fields}\n" + prefix += f"\n{defined_raw_log_fields}" result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) except StrictPlatformException as err: errors.append(err) @@ -407,6 +449,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index 45486ef1..ff9385ba 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Any, ClassVar, Optional, Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -29,18 +30,18 @@ UnsupportedOperatorException, ) from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function from app.translator.core.models.functions.eval import EvalArg from app.translator.core.models.functions.group_by import GroupByFunction +from app.translator.core.models.functions.join import JoinFunction from app.translator.core.models.functions.rename import RenameArg from app.translator.core.models.functions.sort import SortArg +from app.translator.core.models.functions.union import UnionFunction from app.translator.core.models.identifier import Identifier from app.translator.core.str_value_manager import StrValue, StrValueManager from app.translator.tools.utils import get_match_group -TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field] - class BaseTokenizer(ABC): @abstractmethod @@ -323,20 +324,27 @@ def filter_tokens( ) -> list[TOKEN_TYPE]: return [token for token in tokens if isinstance(token, token_type)] - def get_field_tokens_from_func_args( + def get_field_tokens_from_func_args( # noqa: PLR0912 self, args: list[Union[Field, FieldValue, Keyword, Identifier, Function, SortArg]] ) -> list[Field]: result = [] for arg in args: if isinstance(arg, Field): result.append(arg) + elif isinstance(arg, FieldField): + if arg.field_left: + result.append(arg.field_left) + if arg.field_right: + result.append(arg.field_right) elif isinstance(arg, FieldValue): - if not arg.alias or arg.alias.name != arg.field.source_name: + if arg.field: result.append(arg.field) elif isinstance(arg, GroupByFunction): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_])) + elif isinstance(arg, (JoinFunction, UnionFunction)): + continue elif isinstance(arg, Function): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) elif isinstance(arg, SortArg) and isinstance(arg.field, Field): diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml index f6b25023..fa904aaf 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml @@ -125,3 +125,4 @@ field_mapping: SourceOS: xdm.source.host.os DestinationOS: xdm.target.host.os url_category: xdm.network.http.url_category + EventSeverity: xdm.alert.severity diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml index e489fd50..e279a60a 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml @@ -10,4 +10,5 @@ field_mapping: #dns-record: dns-record dns_query_name: xdm.network.dns.dns_question.name QueryName: xdm.network.dns.dns_question.name - query: xdm.network.dns.dns_question.name \ No newline at end of file + query: xdm.network.dns.dns_question.name + dns-record-type: xdm.network.dns.dns_question.type \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/linux_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/linux_process_creation.yml index f1cda96d..06d225bc 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/linux_process_creation.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/linux_process_creation.yml @@ -26,4 +26,5 @@ field_mapping: ParentProduct: actor_process_signature_product ParentCompany: actor_process_signature_vendor md5: action_process_image_md5 - sha256: action_process_image_sha256 \ No newline at end of file + sha256: action_process_image_sha256 + EventID: action_evtlog_event_id \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml index 86110049..04abb36b 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml @@ -28,4 +28,5 @@ field_mapping: ParentIntegrityLevel: causality_actor_process_integrity_level ParentLogonId: causality_actor_process_logon_id ParentProduct: causality_actor_process_signature_product - ParentCompany: causality_actor_process_signature_vendor \ No newline at end of file + ParentCompany: causality_actor_process_signature_vendor + EventType: event_sub_type \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml index 6e798034..00dcef55 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml @@ -13,9 +13,12 @@ field_mapping: dst-port: - DstPort - DestinationPort + - remoteport dst-hostname: DstHost src-hostname: SrcHost - src-port: SourcePort + src-port: + - SourcePort + - localport src-ip: - sourceip - source_ip @@ -27,11 +30,14 @@ field_mapping: - destination_ip - destinationIP - destinationaddress + - destination User: - userName - EventUserName CommandLine: Command - Protocol: IPProtocol + Protocol: + - IPProtocol + - protocol Application: - Application - application @@ -53,10 +59,13 @@ field_mapping: - dst-packets src-bytes: src-bytes dst-bytes: dst-bytes - ExternalSeverity: External Severity + ExternalSeverity: + - External Severity + - Observeit Severity SourceMAC: - SourceMAC - MAC + - sourceMAC DestinationMAC: DestinationMAC SourceOS: - SourceOS @@ -64,4 +73,8 @@ field_mapping: DestinationOS: DestinationOS TargetUserName: DestinationUserName SourceUserName: SourceUserName - url_category: XForceCategoryByURL \ No newline at end of file + url_category: XForceCategoryByURL + EventSeverity: EventSeverity + Source: + - Source + - source \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml b/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml index 048a4bd3..d9aad78e 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml @@ -12,4 +12,5 @@ field_mapping: dns-query: URL parent-domain: parent-domain dns-answer: dns-answer - dns-record: URL \ No newline at end of file + dns-record: URL + dns-record-type: DNSRecordType \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml b/uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml index 14d7aefc..e1313d6d 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/firewall.yml @@ -11,10 +11,13 @@ default_log_source: field_mapping: src-ip: - sourceip + - sourceIP + - SourceIP - SrcHost - LocalHost - Source - NetworkView + - HostName src-port: - sourceport - SrcPort diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml index 80814237..8fddefd6 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml @@ -11,9 +11,12 @@ default_log_source: category: 8110 field_mapping: - CommandLine: Command + CommandLine: + - Command + - ASACommand Image: Process Path ParentCommandLine: Parent Command ParentImage: Parent Process Path User: username - LogonId: Logon ID \ No newline at end of file + LogonId: Logon ID + EventID: ASASyslogCode \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml index 58393ac0..193bc79c 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml @@ -24,6 +24,7 @@ field_mapping: cs-host: - UrlHost - URL Host + - URL Domain cs-referrer: - URL Referrer - Referrer URL diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml index c6bff8b8..1886343a 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml @@ -11,13 +11,20 @@ default_log_source: category: 8110 field_mapping: - CommandLine: Command + CommandLine: + - Command + - Encoded Argument CurrentDirectory: CurrentDirectory Hashes: File Hash - Image: Process Path + Image: + - Process Path + - Process Name + - DGApplication IntegrityLevel: IntegrityLevel ParentCommandLine: Parent Command ParentImage: Parent Process Path ParentUser: ParentUser Product: Product - User: username \ No newline at end of file + User: + - username + - userName \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_termination.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_termination.yml index 563403a4..0109186c 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_termination.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_termination.yml @@ -11,6 +11,8 @@ default_log_source: category: 8113 field_mapping: - Image: Process Path + Image: + - Process Path + - Terminated Process Name ProcessId: ProcessId # ProcessGuid: ProcessGuid \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml index 7d01b97e..9ccb1fbe 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml @@ -9,7 +9,9 @@ default_log_source: devicetype: 12 field_mapping: - EventID: Event ID + EventID: + - Event ID + - EventID ParentImage: Parent Process Path AccessMask: AccessMask AccountName: Account Name @@ -22,13 +24,16 @@ field_mapping: ComputerName: - Machine Identifier - Hostname + - identityNetBiosName EventType: EventType FailureReason: FailureReason FileName: Filename GrantedAccess: GrantedAccess Hashes: File Hash HiveName: HiveName - IpAddress: + IpAddress: + - sourceIP + - SourceIP - sourceip - identityIP IpPort: sourceport @@ -41,9 +46,11 @@ field_mapping: LinkName: LinkName MemberName: MemberName MemberSid: MemberSid - NewProcessName: Process Name + NewProcessName: + - Process Name + - New Process Name ObjectClass: ObjectClass - ObjectName: + ObjectName: - Object Name - objectname - MSFileObjectName @@ -74,6 +81,7 @@ field_mapping: GroupMembership: - GroupMembership - GroupName + - Group Name FilterName: FilterName ChangeType: ChangeType LayerName: LayerName @@ -93,7 +101,9 @@ field_mapping: TargetServerName: TargetServerName NewTargetUserName: NewTargetUserName OperationType: OperationType - DestPort: destinationport + DestPort: + - destinationport + - DstPort ServiceStartType: ServiceStartType OldTargetUserName: OldTargetUserName UserPrincipalName: UserPrincipalName @@ -102,7 +112,10 @@ field_mapping: DisableIntegrityChecks: DisableIntegrityChecks AuditSourceName: AuditSourceName Workstation: Machine Identifier - DestAddress: destinationip + DestAddress: + - destinationip + - DestinationIP + - destinationaddress PreAuthType: PreAuthType SecurityPackageName: SecurityPackageName SubjectLogonId: SubjectLogonId @@ -122,6 +135,7 @@ field_mapping: ServiceFileName: - Service Filename - ServiceFileName + - Service File Name SecurityDescriptor: SecurityDescriptor ServiceName: Service Name ShareName: @@ -147,6 +161,8 @@ field_mapping: TargetSid: TargetSid TargetUserName: - Target Username + - User + - userName - Target User Name ObjectServer: ObjectServer TargetUserSid: TargetUserSid diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml b/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml index acd62dbc..96bb06b8 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml @@ -3,10 +3,10 @@ source: aws_cloudtrail log_source: - source_type: [aws:cloudtrail] + sourcetype: [aws:cloudtrail] default_log_source: - source_type: aws:cloudtrail + sourcetype: aws:cloudtrail field_mapping: eventSource: eventSource diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml b/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml index 32302e30..38e225d7 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml @@ -3,10 +3,10 @@ source: aws_eks log_source: - source_type: [aws:*] + sourcetype: [aws:*] default_log_source: - source_type: aws:* + sourcetype: aws:* field_mapping: annotations.authorization.k8s.io\/decision: annotations.authorization.k8s.io\/decision diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml index 5cff60da..90fd75a1 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml @@ -3,10 +3,10 @@ source: azure_AzureDiagnostics log_source: - source_type: [azure:*] + sourcetype: [azure:*] default_log_source: - source_type: azure:* + sourcetype: azure:* field_mapping: ResultDescription: ResultDescription diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml index 379004da..e1f17620 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml @@ -3,10 +3,10 @@ source: azure_BehaviorAnalytics log_source: - source_type: [azure:*] + sourcetype: [azure:*] default_log_source: - source_type: azure:* + sourcetype: azure:* field_mapping: ActionType: ActionType diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml index 3e994fc5..ad6bb5eb 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml @@ -3,10 +3,10 @@ source: azure_aadnoninteractiveusersigninlogs log_source: - source_type: [azure:*] + sourcetype: [azure:*] default_log_source: - source_type: azure:* + sourcetype: azure:* field_mapping: UserAgent: UserAgent diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml index d3623983..337125f4 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml @@ -3,10 +3,10 @@ source: azure_azureactivity log_source: - source_type: [mscs:azure:*, azure:*] + sourcetype: [mscs:azure:*, azure:*] default_log_source: - source_type: mscs:azure:* + sourcetype: mscs:azure:* field_mapping: ActivityStatus: ActivityStatus diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml index 5f393c91..69e3d195 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml @@ -3,10 +3,10 @@ source: azure_azuread log_source: - source_type: [azure:aad:*] + sourcetype: [azure:aad:*] default_log_source: - source_type: azure:aad:* + sourcetype: azure:aad:* field_mapping: ActivityDisplayName: ActivityDisplayName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml index 23b7569b..4f669d89 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml @@ -3,10 +3,10 @@ source: azure_signinlogs log_source: - source_type: [azure:aad:*] + sourcetype: [azure:aad:*] default_log_source: - source_type: azure:aad:* + sourcetype: azure:aad:* field_mapping: AppDisplayName: AppDisplayName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml b/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml index f40ef682..ed886d9c 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml @@ -3,11 +3,11 @@ source: firewall log_source: - source_type: [fortigate_traffic] + sourcetype: [fortigate_traffic] index: [fortigate] default_log_source: - source_type: fortigate_traffic + sourcetype: fortigate_traffic index: fortigate field_mapping: diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml index ef92fb58..be54b882 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml @@ -3,7 +3,7 @@ source: gcp_gcp.audit log_source: - source_type: [google:gcp:*] + sourcetype: [google:gcp:*] default_log_source: index: google:gcp:* diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml index 7ab8483c..dbfd2736 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml @@ -3,7 +3,7 @@ source: gcp_pubsub log_source: - source_type: [google:gcp:*] + sourcetype: [google:gcp:*] default_log_source: index: google:gcp:* diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml b/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml index ee3ac161..afd115b0 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml @@ -3,10 +3,10 @@ source: linux_auditd log_source: - source_type: [linux:audit] + sourcetype: [linux:audit] default_log_source: - source_type: linux:audit + sourcetype: linux:audit field_mapping: a0: a0 diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml b/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml index 3ee6d0e1..3f55621f 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml @@ -3,10 +3,10 @@ source: okta_okta log_source: - source_type: [OktaIM2:*] + sourcetype: [OktaIM2:*] default_log_source: - source_type: OktaIM2:* + sourcetype: OktaIM2:* field_mapping: client.user.id: client.user.id diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml index 014287eb..babbd610 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml @@ -2,10 +2,10 @@ platform: Splunk source: windows_bits_client log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational field_mapping: LocalName: LocalName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml index 698e62cc..d8e40100 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml @@ -4,11 +4,11 @@ source: windows_dns_query log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: Image: Image diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml index f8248b8e..86b76510 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml @@ -4,11 +4,11 @@ source: windows_driver_load log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: ImageLoaded: ImageLoaded diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml index 5c1c64f2..48ab5786 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml @@ -4,11 +4,11 @@ source: windows_file_access log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml index 0114b7e0..f45393aa 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml @@ -4,11 +4,11 @@ source: windows_file_change log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml index d9b0d8c0..485ea463 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml @@ -4,11 +4,11 @@ source: windows_file_create log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml index 8b82cc38..13660235 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml @@ -4,11 +4,11 @@ source: windows_file_delete log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml index 278b9b30..ed0855d3 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml @@ -4,11 +4,11 @@ source: windows_file_event log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml index 10390535..dae50085 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml @@ -4,11 +4,11 @@ source: windows_file_rename log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml index 8f427639..3cc22f55 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml @@ -4,11 +4,11 @@ source: windows_image_load log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: Image: Image diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml index 8fc85d34..f8241ba4 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml @@ -3,10 +3,10 @@ source: windows_ldap_debug log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug] + sourcetype: [XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug + sourcetype: XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug field_mapping: EventID: EventID diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml index d8260810..7a92b32c 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml @@ -4,11 +4,11 @@ source: windows_network_connection log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: Image: Image diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml index 3ea2c8ea..7902c0fe 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml @@ -3,10 +3,10 @@ source: windows_ntlm log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-NTLM/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-NTLM/Operational] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-NTLM/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-NTLM/Operational field_mapping: WorkstationName: WorkstationName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml index 8cbe38f3..a2169567 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml @@ -4,11 +4,11 @@ source: windows_registry_event log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: TargetObject: TargetObject diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml index a361471a..89bf98e0 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml @@ -3,11 +3,11 @@ source: windows_sysmon log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CommandLine: CommandLine diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml index 5e1e47bd..b1e415d0 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml @@ -3,10 +3,10 @@ source: windows_wmi_event log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational field_mapping: Destination: Destination diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena.py b/uncoder-core/app/translator/platforms/athena/renders/athena.py index a62e5b00..8550c94a 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena.py @@ -21,10 +21,10 @@ from app.translator.managers import render_manager from app.translator.platforms.athena.const import athena_details from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender -class AthenaFieldValue(SqlFieldValue): +class AthenaFieldValueRender(SqlFieldValueRender): details: PlatformDetails = athena_details @@ -35,7 +35,7 @@ class AthenaQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = AthenaFieldValue(or_token=or_token) + field_value_render = AthenaFieldValueRender(or_token=or_token) comment_symbol = "--" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py index 05826d08..6c0c1665 100644 --- a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager -class AQLFieldValue(BaseQueryFieldValue): +class AQLFieldValueRender(BaseFieldValueRender): str_value_manager = aql_str_value_manager @staticmethod @@ -127,8 +127,6 @@ class AQLQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = AQLFieldValue(or_token=or_token) - def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) extra_condition = log_source_signature.extra_condition diff --git a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py index b5994499..f8511d82 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.lucene.mapping import LuceneLogSourceSignature from app.translator.platforms.base.lucene.str_value_manager import lucene_str_value_manager -class LuceneFieldValue(BaseQueryFieldValue): +class LuceneFieldValueRender(BaseFieldValueRender): str_value_manager = lucene_str_value_manager @staticmethod diff --git a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py index b2c12068..74adf32b 100644 --- a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py @@ -21,11 +21,11 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.platforms.base.spl.escape_manager import spl_escape_manager -class SplFieldValue(BaseQueryFieldValue): +class SplFieldValueRender(BaseFieldValueRender): escape_manager = spl_escape_manager def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: diff --git a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py index 43904a1e..d69f1590 100644 --- a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py @@ -22,10 +22,10 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import LogSourceSignature -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender -class SqlFieldValue(BaseQueryFieldValue): +class SqlFieldValueRender(BaseFieldValueRender): def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py index 4101b825..8bcbe56f 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py @@ -23,14 +23,14 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import chronicle_query_details from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings -class ChronicleFieldValue(BaseQueryFieldValue): +class ChronicleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = chronicle_query_details escape_manager = chronicle_escape_manager @@ -109,6 +109,6 @@ class ChronicleQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = ChronicleFieldValue(or_token=or_token) + field_value_render = ChronicleFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py index aaa64384..1961c72b 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py @@ -26,12 +26,12 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details -from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValue, ChronicleQueryRender +from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValueRender, ChronicleQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated Chronicle Security rule." -class ChronicleRuleFieldValue(ChronicleFieldValue): +class ChronicleRuleFieldValueRender(ChronicleFieldValueRender): details: PlatformDetails = chronicle_rule_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -85,7 +85,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: class ChronicleSecurityRuleRender(ChronicleQueryRender): details: PlatformDetails = chronicle_rule_details or_token = "or" - field_value_map = ChronicleRuleFieldValue(or_token=or_token) + field_value_render = ChronicleRuleFieldValueRender(or_token=or_token) @staticmethod def prepare_title(title: str) -> str: diff --git a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py index 8c6630e9..3e5900cc 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings -class CrowdStrikeFieldValue(SplFieldValue): +class CrowdStrikeFieldValueRender(SplFieldValueRender): details = crowdstrike_query_details @@ -36,7 +36,7 @@ class CrowdStrikeQueryRender(SplQueryRender): platform_functions: CrowdStrikeFunctions = None or_token = "OR" - field_value_map = CrowdStrikeFieldValue(or_token=or_token) + field_value_render = CrowdStrikeFieldValueRender(or_token=or_token) comment_symbol = "`" def init_platform_functions(self) -> None: diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py index 09fad79b..0b7b20c4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -50,7 +50,7 @@ class ElasticSearchRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticSearchRuleFieldValue(or_token=or_token) + field_value_render = ElasticSearchRuleFieldValue(or_token=or_token) def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): @@ -109,7 +109,4 @@ def finalize_query( } ) rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py index 104b8ecc..9d7914ab 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py @@ -49,7 +49,7 @@ class ElastAlertRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticAlertRuleFieldValue(or_token=or_token) + field_value_render = ElasticAlertRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -75,7 +75,4 @@ def finalize_query( ) rule = rule.replace("", meta_info.title or _AUTOGENERATED_TEMPLATE) rule = rule.replace("", _SEVERITIES_MAP[meta_info.severity]) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py index 8d2db1d0..2e6a12f0 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -class ElasticSearchFieldValue(LuceneFieldValue): +class ElasticSearchFieldValue(LuceneFieldValueRender): details: PlatformDetails = elasticsearch_lucene_query_details @@ -34,4 +34,4 @@ class ElasticSearchQueryRender(LuceneQueryRender): mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = ElasticSearchFieldValue(or_token=or_token) + field_value_render = ElasticSearchFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py index c3b6a46a..53a4acf5 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py @@ -45,7 +45,7 @@ class KibanaRuleRender(ElasticSearchQueryRender): details: PlatformDetails = kibana_rule_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = KibanaFieldValue(or_token=or_token) + field_value_render = KibanaFieldValue(or_token=or_token) def finalize_query( self, @@ -74,7 +74,4 @@ def finalize_query( references=meta_info.references, ) rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py index 9a013dcf..d8421977 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py @@ -45,7 +45,7 @@ class XPackWatcherRuleRender(ElasticSearchQueryRender): details: PlatformDetails = xpack_watcher_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = XpackWatcherRuleFieldValue(or_token=or_token) + field_value_render = XpackWatcherRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -78,7 +78,4 @@ def finalize_query( rule["input"]["search"]["request"]["indices"] = indices rule["actions"]["send_email"]["email"]["subject"] = meta_info.title or _AUTOGENERATED_TEMPLATE rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index 65ca0b07..dfbc2ee6 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -18,6 +18,7 @@ from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.const import TOKEN_TYPE from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType @@ -28,9 +29,8 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue -from app.translator.core.tokenizer import TOKEN_TYPE from app.translator.managers import render_manager from app.translator.platforms.forti_siem.const import ( FORTI_SIEM_RULE, @@ -76,7 +76,7 @@ ] -class FortiSiemFieldValue(BaseQueryFieldValue): +class FortiSiemFieldValueRender(BaseFieldValueRender): details: PlatformDetails = forti_siem_rule_details str_value_manager = forti_siem_str_value_manager @@ -194,7 +194,7 @@ class FortiSiemRuleRender(PlatformQueryRender): group_token = "(%s)" - field_value_map = FortiSiemFieldValue(or_token=or_token) + field_value_render = FortiSiemFieldValueRender(or_token=or_token) @staticmethod def __is_negated_token(prev_token: TOKEN_TYPE) -> bool: @@ -244,7 +244,7 @@ def __replace_not_tokens(self, tokens: list[TOKEN_TYPE]) -> list[TOKEN_TYPE]: return tokens - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -324,11 +324,7 @@ def finalize_query( rule = rule.replace("", query) rule = rule.replace("", ", ".join(args_list)) rule = rule.replace("", self.get_attr_str(fields.copy())) - - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) @staticmethod def get_attr_str(fields: set[str]) -> str: diff --git a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py index 2bdf001e..986ddd93 100644 --- a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py +++ b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.graylog.const import graylog_details from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings -class GraylogFieldValue(LuceneFieldValue): +class GraylogFieldValue(LuceneFieldValueRender): details: PlatformDetails = graylog_details @@ -34,4 +34,4 @@ class GraylogQueryRender(LuceneQueryRender): mappings: GraylogMappings = graylog_mappings or_token = "OR" - field_value_map = GraylogFieldValue(or_token=or_token) + field_value_render = GraylogFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py index 0348bfb0..3c73c234 100644 --- a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py +++ b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender from app.translator.platforms.hunters.const import hunters_details from app.translator.platforms.hunters.mapping import HuntersMappings, hunters_mappings -class HuntersFieldValue(SqlFieldValue): +class HuntersFieldValueRender(SqlFieldValueRender): details: PlatformDetails = hunters_details @@ -35,7 +35,7 @@ class HuntersQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = HuntersFieldValue(or_token=or_token) + field_value_render = HuntersFieldValueRender(or_token=or_token) @staticmethod def _finalize_search_query(query: str) -> str: diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 624fa3d7..95dbc40a 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -30,7 +30,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_axon_query_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_query_escape_manager @@ -41,7 +41,7 @@ class LogRhythmRegexRenderException(BaseRenderException): ... -class LogRhythmAxonFieldValue(BaseQueryFieldValue): +class LogRhythmAxonFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logrhythm_axon_query_details escape_manager = logrhythm_query_escape_manager @@ -204,7 +204,7 @@ class LogRhythmAxonQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = LogRhythmAxonFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonFieldValueRender(or_token=or_token) mappings: LogRhythmAxonMappings = logrhythm_axon_mappings comment_symbol = "//" @@ -219,12 +219,12 @@ def generate_prefix(self, log_source_signature: LogSourceSignature, functions_pr return str(log_source_signature) def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: - if isinstance(token, FieldValue): + if isinstance(token, FieldValue) and token.field: try: mapped_fields = self.map_field(token.field, source_mapping) except StrictPlatformException: try: - return self.field_value_map.apply_field_value( + return self.field_value_render.apply_field_value( field=UNMAPPED_FIELD_DEFAULT_NAME, operator=token.operator, value=token.value ) except LogRhythmRegexRenderException as exc: @@ -232,20 +232,17 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp f"Uncoder does not support complex regexp for unmapped field:" f" {token.field.source_name} for LogRhythm Axon" ) from exc - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value(field=field, operator=token.operator, value=token.value) - for field in mapped_fields - ] - ) - return self.field_value_map.apply_field_value( - field=mapped_fields[0], operator=token.operator, value=token.value + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] ) + return self.group_token % joined if len(mapped_fields) > 1 else joined return super().apply_token(token, source_mapping) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py index 20514140..2e68c2d1 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py @@ -29,7 +29,7 @@ from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import ( - LogRhythmAxonFieldValue, + LogRhythmAxonFieldValueRender, LogRhythmAxonQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -44,7 +44,7 @@ } -class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): +class LogRhythmAxonRuleFieldValueRender(LogRhythmAxonFieldValueRender): details: PlatformDetails = logrhythm_axon_rule_details escape_manager = logrhythm_rule_escape_manager @@ -53,7 +53,7 @@ class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): class LogRhythmAxonRuleRender(LogRhythmAxonQueryRender): details: PlatformDetails = logrhythm_axon_rule_details or_token = "or" - field_value_map = LogRhythmAxonRuleFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonRuleFieldValueRender(or_token=or_token) def finalize_query( self, @@ -93,7 +93,4 @@ def finalize_query( ] json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py index 9cb7cf05..e1ed4818 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py @@ -23,7 +23,7 @@ from app.translator.core.mapping import SourceMapping from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.escape_manager import logscale_escape_manager @@ -31,7 +31,7 @@ from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings -class LogScaleFieldValue(BaseQueryFieldValue): +class LogScaleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logscale_query_details escape_manager = logscale_escape_manager @@ -102,7 +102,7 @@ class LogScaleQueryRender(PlatformQueryRender): and_token = "" not_token = "not" - field_value_map = LogScaleFieldValue(or_token=or_token) + field_value_render = LogScaleFieldValueRender(or_token=or_token) def init_platform_functions(self) -> None: self.platform_functions = log_scale_functions @@ -123,8 +123,5 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = super().finalize_query(prefix=prefix, query=query, functions=functions) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py index 4b3af0fb..a6628045 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py @@ -26,13 +26,13 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details -from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValue, LogScaleQueryRender +from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValueRender, LogScaleQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Falcon LogScale Alert" -class LogScaleAlertFieldValue(LogScaleFieldValue): +class LogScaleAlertFieldValueRender(LogScaleFieldValueRender): details: PlatformDetails = logscale_alert_details @@ -40,7 +40,7 @@ class LogScaleAlertFieldValue(LogScaleFieldValue): class LogScaleAlertRender(LogScaleQueryRender): details: PlatformDetails = logscale_alert_details or_token = "or" - field_value_map = LogScaleAlertFieldValue(or_token=or_token) + field_value_render = LogScaleAlertFieldValueRender(or_token=or_token) def finalize_query( self, @@ -70,8 +70,5 @@ def finalize_query( mitre_attack=mitre_attack, ) - json_query = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_query + rendered_not_supported - return json_query + rule_str = json.dumps(rule, indent=4, sort_keys=False) + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py index 7b7a3779..38617b55 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py @@ -23,12 +23,12 @@ from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) -class MicrosoftDefenderFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftDefenderFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_defender_details @@ -38,7 +38,7 @@ class MicrosoftDefenderQueryRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_defender_details platform_functions: MicrosoftFunctions = None or_token = "or" - field_value_map = MicrosoftDefenderFieldValue(or_token=or_token) + field_value_render = MicrosoftDefenderFieldValueRender(or_token=or_token) is_strict_mapping = True diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py index 3153f8d4..7ef6f1f9 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py @@ -22,7 +22,7 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.escape_manager import microsoft_escape_manager @@ -30,7 +30,7 @@ from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings -class MicrosoftSentinelFieldValue(BaseQueryFieldValue): +class MicrosoftSentinelFieldValueRender(BaseFieldValueRender): details: PlatformDetails = microsoft_sentinel_query_details escape_manager = microsoft_escape_manager @@ -128,7 +128,7 @@ class MicrosoftSentinelQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = MicrosoftSentinelFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelFieldValueRender(or_token=or_token) mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings comment_symbol = "//" diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index e2fdb81f..b5631ef5 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -28,7 +28,7 @@ from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -42,7 +42,7 @@ } -class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftSentinelRuleFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_sentinel_rule_details @@ -50,7 +50,7 @@ class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): class MicrosoftSentinelRuleRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_sentinel_rule_details or_token = "or" - field_value_map = MicrosoftSentinelRuleFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelRuleFieldValueRender(or_token=or_token) def __create_mitre_threat(self, meta_info: MetaInfoContainer) -> tuple[list, list]: tactics = set() @@ -92,7 +92,4 @@ def finalize_query( rule["tactics"] = mitre_tactics rule["techniques"] = mitre_techniques json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py index 1d2145a7..3298c106 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py @@ -24,12 +24,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.opensearch.const import opensearch_query_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -class OpenSearchFieldValue(LuceneFieldValue): +class OpenSearchFieldValueRender(LuceneFieldValueRender): details: PlatformDetails = opensearch_query_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -102,4 +102,4 @@ class OpenSearchQueryRender(LuceneQueryRender): mappings: OpenSearchMappings = opensearch_mappings or_token = "OR" - field_value_map = OpenSearchFieldValue(or_token=or_token) + field_value_render = OpenSearchFieldValueRender(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py index 3f68e6c6..09cd5b62 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -30,13 +30,13 @@ from app.translator.managers import render_manager from app.translator.platforms.opensearch.const import OPENSEARCH_RULE, opensearch_rule_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValue, OpenSearchQueryRender +from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValueRender, OpenSearchQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated AWS OpenSearch Rule" _SEVERITIES_MAP = {SeverityType.critical: "5", SeverityType.high: "4", SeverityType.medium: "3", SeverityType.low: "2"} -class OpenSearchRuleFieldValue(OpenSearchFieldValue): +class OpenSearchRuleFieldValueRender(OpenSearchFieldValueRender): details: PlatformDetails = opensearch_rule_details @@ -49,7 +49,7 @@ class OpenSearchRuleRender(OpenSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = OpenSearchRuleFieldValue(or_token=or_token) + field_value_render = OpenSearchRuleFieldValueRender(or_token=or_token) def __init__(self): super().__init__() @@ -76,13 +76,10 @@ def finalize_query( rule["triggers"][0]["severity"] = _SEVERITIES_MAP[meta_info.severity] rule["triggers"][0]["actions"][0]["message_template"]["source"] = str(source).replace(", ", ",\n") rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: - if isinstance(token, FieldValue): + if isinstance(token, FieldValue) and token.field: for field in self.map_field(token.field, source_mapping): self.fields.update({field: f"{{ctx.results.0.hits.hits.0._source.{field}}}"}) return super().apply_token(token, source_mapping) diff --git a/uncoder-core/app/translator/platforms/palo_alto/const.py b/uncoder-core/app/translator/platforms/palo_alto/const.py index 4b94fea8..2cff5d5b 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/const.py +++ b/uncoder-core/app/translator/platforms/palo_alto/const.py @@ -1,3 +1,4 @@ +from app.translator.core.custom_types.predefined_fields import IPLocationType from app.translator.core.models.platform_details import PlatformDetails PLATFORM_DETAILS = {"group_id": "cortex", "group_name": "Palo Alto Cortex XSIAM"} @@ -10,3 +11,15 @@ } cortex_xql_query_details = PlatformDetails(**CORTEX_XSIAM_XQL_QUERY_DETAILS) + + +PREDEFINED_FIELDS_MAP = { + IPLocationType.asn: "loc_asn", + IPLocationType.asn_org: "loc_asn_org", + IPLocationType.city: "loc_city", + IPLocationType.continent: "loc_continent", + IPLocationType.country: "loc_country", + IPLocationType.lat_lon: "loc_latlon", + IPLocationType.region: "loc_region", + IPLocationType.timezone: "loc_timezone", +} diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index 72a2737b..31bdd1e0 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -20,12 +20,17 @@ from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import preset_log_source_str_ctx_var +from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType +from app.translator.core.mapping import SourceMapping +from app.translator.core.models.field import FieldValue, Keyword +from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager -from app.translator.platforms.palo_alto.const import cortex_xql_query_details +from app.translator.platforms.palo_alto.const import PREDEFINED_FIELDS_MAP, cortex_xql_query_details from app.translator.platforms.palo_alto.functions import CortexXQLFunctions, cortex_xql_functions from app.translator.platforms.palo_alto.mapping import ( CortexXQLLogSourceSignature, @@ -34,8 +39,18 @@ ) from app.translator.platforms.palo_alto.str_value_manager import cortex_xql_str_value_manager +SOURCE_MAPPING_TO_FIELD_VALUE_MAP = { + "windows_registry_event": { + "EventType": { + "SetValue": "REGISTRY_SET_VALUE", + "DeleteValue": "REGISTRY_DELETE_VALUE", + "CreateKey": "REGISTRY_CREATE_KEY", + } + } +} + -class CortexXQLFieldValue(BaseQueryFieldValue): +class CortexXQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = cortex_xql_query_details str_value_manager = cortex_xql_str_value_manager @@ -56,7 +71,8 @@ def _wrap_str_value(value: str) -> str: def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): values = ", ".join( - f"{self._pre_process_value(field, v, value_type=ValueType.value, wrap_str=True)}" for v in value + f"{self._pre_process_value(field, str(v) if isinstance(v, int) else v, ValueType.value, True)}" + for v in value ) return f"{field} in ({values})" @@ -107,7 +123,11 @@ def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" - return f"{field} ~= {self._pre_process_value(field ,value, value_type=ValueType.regex_value, wrap_str=True)}" + value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=True) + if value.endswith('\\\\"'): + value = value[:-1] + "]" + value[-1:] + value = value[:-4] + "[" + value[-4:] + return f"{field} ~= {value}" def not_regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): @@ -132,12 +152,24 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"_raw_log contains {self._pre_process_value(field ,value, value_type=ValueType.value, wrap_str=True)}" +class CortexXQLFieldFieldRender(BaseFieldFieldRender): + operators_map: ClassVar[dict[str, str]] = { + OperatorType.EQ: "=", + OperatorType.NOT_EQ: "!=", + OperatorType.LT: "<", + OperatorType.LTE: "<=", + OperatorType.GT: ">", + OperatorType.GTE: ">=", + } + + @render_manager.register class CortexXQLQueryRender(PlatformQueryRender): details: PlatformDetails = cortex_xql_query_details mappings: CortexXQLMappings = cortex_xql_mappings is_strict_mapping = True - raw_log_field_pattern_map: ClassVar[dict[str, str]] = { + predefined_fields_map = PREDEFINED_FIELDS_MAP + raw_log_field_patterns_map: ClassVar[dict[str, str]] = { "regex": '| alter {field} = regextract(to_json_string(action_evtlog_data_fields)->{field}{{}}, "\\"(.*)\\"")', "object": '| alter {field_name} = json_extract_scalar({field_object} , "$.{field_path}")', "list": '| alter {field_name} = arraystring(json_extract_array({field_object} , "$.{field_path}")," ")', @@ -147,8 +179,10 @@ class CortexXQLQueryRender(PlatformQueryRender): or_token = "or" and_token = "and" not_token = "not" + query_parts_delimiter = "\n" - field_value_map = CortexXQLFieldValue(or_token=or_token) + field_field_render = CortexXQLFieldFieldRender() + field_value_render = CortexXQLFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = False @@ -157,7 +191,7 @@ def init_platform_functions(self) -> None: self.platform_functions.platform_query_render = self def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: - raw_log_field_pattern = self.raw_log_field_pattern_map.get(field_type) + raw_log_field_pattern = self.raw_log_field_patterns_map.get(field_type) if raw_log_field_pattern is None: return if field_type == "regex": @@ -170,7 +204,21 @@ def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, functions_prefix: str = "") -> str: functions_prefix = f"{functions_prefix} | " if functions_prefix else "" - return f"{functions_prefix}{log_source_signature}" + log_source_str = preset_log_source_str_ctx_var.get() or str(log_source_signature) + return f"{functions_prefix}{log_source_str}" + + def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + if isinstance(token, FieldValue) and token.field: + field_name = token.field.source_name + if values_map := SOURCE_MAPPING_TO_FIELD_VALUE_MAP.get(source_mapping.source_id, {}).get(field_name): + values_to_update = [] + for token_value in token.values: + mapped_value: str = values_map.get(token_value, token_value) + values_to_update.append( + StrValue(value=mapped_value, split_value=mapped_value.split()) if mapped_value else token_value + ) + token.value = values_to_update + return super().apply_token(token=token, source_mapping=source_mapping) @staticmethod def _finalize_search_query(query: str) -> str: diff --git a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py index 0f06fb40..cf4a7d51 100644 --- a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py @@ -19,14 +19,15 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.aql.renders.aql import AQLFieldValue, AQLQueryRender +from app.translator.platforms.base.aql.renders.aql import AQLFieldValueRender, AQLQueryRender from app.translator.platforms.qradar.const import qradar_query_details -class QradarFieldValue(AQLFieldValue): +class QradarFieldValueRender(AQLFieldValueRender): details: PlatformDetails = qradar_query_details @render_manager.register class QradarQueryRender(AQLQueryRender): details: PlatformDetails = qradar_query_details + field_value_render = QradarFieldValueRender(or_token=AQLQueryRender.or_token) diff --git a/uncoder-core/app/translator/platforms/sigma/mapping.py b/uncoder-core/app/translator/platforms/sigma/mapping.py index 3f23700d..1af791ac 100644 --- a/uncoder-core/app/translator/platforms/sigma/mapping.py +++ b/uncoder-core/app/translator/platforms/sigma/mapping.py @@ -19,9 +19,9 @@ def __init__( def is_suitable( self, service: Optional[list[str]], product: Optional[list[str]], category: Optional[list[str]] ) -> bool: - product_match = set(product or []).issubset(self.products) if product else False - category_match = set(category or []).issubset(self.categories) if category else False - service_match = set(service or []).issubset(self.services) if service else False + product_match = set(product_.lower() for product_ in product or []).issubset(self.products) if product else False + category_match = set(category_.lower() for category_ in category or []).issubset(self.categories) if category else False + service_match = set(service_.lower() for service_ in service or [] or []).issubset(self.services) if service else False if not product and not service: return category_match return product_match and service_match or product_match and category_match diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index dc33a507..856fd4a3 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -281,10 +281,10 @@ def __get_source_mapping(self, source_mapping_ids: list[str]) -> SourceMapping: return self.mappings.get_source_mapping(DEFAULT_MAPPING_NAME) - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: raise NotImplementedError - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: self.reset_counters() meta_info = query_container.meta_info @@ -316,6 +316,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/platforms/splunk/mapping.py b/uncoder-core/app/translator/platforms/splunk/mapping.py index 2f9c4a8d..1851b8af 100644 --- a/uncoder-core/app/translator/platforms/splunk/mapping.py +++ b/uncoder-core/app/translator/platforms/splunk/mapping.py @@ -42,8 +42,8 @@ def prepare_log_source_signature(self, mapping: dict) -> SplunkLogSourceSignatur default_log_source = mapping["default_log_source"] return SplunkLogSourceSignature( sources=log_source.get("source"), - source_types=log_source.get("source_type"), - source_categories=log_source.get("source_category"), + source_types=log_source.get("sourcetype"), + source_categories=log_source.get("sourcecategory"), indices=log_source.get("index"), default_source=default_log_source, ) diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py index f9404cac..e14c6bfc 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings -class SplunkFieldValue(SplFieldValue): +class SplunkFieldValueRender(SplFieldValueRender): details: PlatformDetails = splunk_query_details @@ -35,7 +35,7 @@ class SplunkQueryRender(SplQueryRender): or_token = "OR" - field_value_map = SplunkFieldValue(or_token=or_token) + field_value_render = SplunkFieldValueRender(or_token=or_token) mappings: SplunkMappings = splunk_mappings platform_functions: SplunkFunctions = None diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py index ef0d097d..5dc2096a 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py @@ -25,14 +25,14 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details -from app.translator.platforms.splunk.renders.splunk import SplunkFieldValue, SplunkQueryRender +from app.translator.platforms.splunk.renders.splunk import SplunkFieldValueRender, SplunkQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Splunk Alert" _SEVERITIES_MAP = {SeverityType.critical: "4", SeverityType.high: "3", SeverityType.medium: "2", SeverityType.low: "1"} -class SplunkAlertFieldValue(SplunkFieldValue): +class SplunkAlertFieldValueRender(SplunkFieldValueRender): details: PlatformDetails = splunk_alert_details @@ -40,7 +40,7 @@ class SplunkAlertFieldValue(SplunkFieldValue): class SplunkAlertRender(SplunkQueryRender): details: PlatformDetails = splunk_alert_details or_token = "OR" - field_value_map = SplunkAlertFieldValue(or_token=or_token) + field_value_render = SplunkAlertFieldValueRender(or_token=or_token) @staticmethod def __create_mitre_threat(meta_info: MetaInfoContainer) -> dict: @@ -74,7 +74,4 @@ def finalize_query( if mitre_techniques: mitre_str = f"action.correlationsearch.annotations = {mitre_techniques})" rule = rule.replace("", mitre_str) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) From 69c5c2a1ad4bc3fd332777bd431df367896ffabd Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 2 Jul 2024 09:55:34 +0200 Subject: [PATCH 18/40] update in render --- .../app/translator/platforms/elasticsearch/renders/esql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 401d0b1e..80b17bdb 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -22,14 +22,14 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings from app.translator.platforms.elasticsearch.str_value_manager import ESQLStrValueManager, esql_str_value_manager -class ESQLFieldValue(BaseQueryFieldValue): +class ESQLFieldValue(BaseFieldValueRender): details: PlatformDetails = elasticsearch_esql_query_details str_value_manager: ESQLStrValueManager = esql_str_value_manager @@ -112,7 +112,7 @@ class ESQLQueryRender(PlatformQueryRender): or_token = "or" and_token = "and" not_token = "not" - field_value_map = ESQLFieldValue(or_token=or_token) + field_value_render = ESQLFieldValue(or_token=or_token) def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) if str(log_source_signature) else "*" From 8e776cc65102529d0485906ff114e06562faab55 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:05:01 +0200 Subject: [PATCH 19/40] merge prod From cb9c1ecf33dc7cf75af77c71b92f5ee14467b870 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:05:05 +0200 Subject: [PATCH 20/40] update in render From 0128b8c6885d9cb12446f868dbf900f3864929a5 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:05:10 +0200 Subject: [PATCH 21/40] renaming --- .../app/translator/platforms/elasticsearch/renders/esql.py | 4 ++-- .../translator/platforms/elasticsearch/renders/esql_rule.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 80b17bdb..b3fcf194 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -29,7 +29,7 @@ from app.translator.platforms.elasticsearch.str_value_manager import ESQLStrValueManager, esql_str_value_manager -class ESQLFieldValue(BaseFieldValueRender): +class ESQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = elasticsearch_esql_query_details str_value_manager: ESQLStrValueManager = esql_str_value_manager @@ -112,7 +112,7 @@ class ESQLQueryRender(PlatformQueryRender): or_token = "or" and_token = "and" not_token = "not" - field_value_render = ESQLFieldValue(or_token=or_token) + field_value_render = ESQLFieldValueRender(or_token=or_token) def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) if str(log_source_signature) else "*" diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py index a118ba3c..ca0c38b1 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -27,12 +27,12 @@ from app.translator.managers import render_manager from app.translator.platforms.elasticsearch.const import ESQL_RULE, elasticsearch_esql_rule_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -from app.translator.platforms.elasticsearch.renders.esql import ESQLFieldValue, ESQLQueryRender +from app.translator.platforms.elasticsearch.renders.esql import ESQLFieldValueRender, ESQLQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated ESQL Rule" -class ESQLRuleFieldValue(ESQLFieldValue): +class ESQLRuleFieldValueRender(ESQLFieldValueRender): details: PlatformDetails = elasticsearch_esql_rule_details From c5198aca47c8478ca327af90f1f6a7628c8208c3 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:17:04 +0200 Subject: [PATCH 22/40] merge prod From 58e7c2e1dd7bb6a0872c017a856cae7a4f5fa9f8 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:17:08 +0200 Subject: [PATCH 23/40] update in render --- .../app/translator/platforms/elasticsearch/renders/esql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index b3fcf194..80b17bdb 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -29,7 +29,7 @@ from app.translator.platforms.elasticsearch.str_value_manager import ESQLStrValueManager, esql_str_value_manager -class ESQLFieldValueRender(BaseFieldValueRender): +class ESQLFieldValue(BaseFieldValueRender): details: PlatformDetails = elasticsearch_esql_query_details str_value_manager: ESQLStrValueManager = esql_str_value_manager @@ -112,7 +112,7 @@ class ESQLQueryRender(PlatformQueryRender): or_token = "or" and_token = "and" not_token = "not" - field_value_render = ESQLFieldValueRender(or_token=or_token) + field_value_render = ESQLFieldValue(or_token=or_token) def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) if str(log_source_signature) else "*" From 07148afeb08ef8ae160f78532985b45694261134 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:17:13 +0200 Subject: [PATCH 24/40] renaming --- .../app/translator/platforms/elasticsearch/renders/esql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 80b17bdb..b3fcf194 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -29,7 +29,7 @@ from app.translator.platforms.elasticsearch.str_value_manager import ESQLStrValueManager, esql_str_value_manager -class ESQLFieldValue(BaseFieldValueRender): +class ESQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = elasticsearch_esql_query_details str_value_manager: ESQLStrValueManager = esql_str_value_manager @@ -112,7 +112,7 @@ class ESQLQueryRender(PlatformQueryRender): or_token = "or" and_token = "and" not_token = "not" - field_value_render = ESQLFieldValue(or_token=or_token) + field_value_render = ESQLFieldValueRender(or_token=or_token) def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) if str(log_source_signature) else "*" From b86cf619729a882023bfd2a21bdb3c55bf29403d Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:17:17 +0200 Subject: [PATCH 25/40] upd --- .../translator/platforms/elasticsearch/renders/esql_rule.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py index ca0c38b1..08edb57a 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -42,6 +42,9 @@ class ESQLRuleRender(ESQLQueryRender): mappings: ElasticSearchMappings = elasticsearch_mappings mitre: MitreConfig = MitreConfig() + or_token = "or" + field_value_render = ESQLRuleFieldValueRender(or_token=or_token) + def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): return [] From c1249e4ee85b8fbe47f4ad551811805aebf1c6bc Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 23 Jul 2024 09:06:53 +0200 Subject: [PATCH 26/40] renaming --- .../translator/platforms/elasticsearch/renders/esql_rule.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py index 08edb57a..ca0c38b1 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -42,9 +42,6 @@ class ESQLRuleRender(ESQLQueryRender): mappings: ElasticSearchMappings = elasticsearch_mappings mitre: MitreConfig = MitreConfig() - or_token = "or" - field_value_render = ESQLRuleFieldValueRender(or_token=or_token) - def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): return [] From fe7a9f4c6f7bea8f53e6b93286aca7d36abc48f4 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 23 Jul 2024 09:06:57 +0200 Subject: [PATCH 27/40] upd --- .../translator/platforms/elasticsearch/renders/esql_rule.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py index ca0c38b1..08edb57a 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -42,6 +42,9 @@ class ESQLRuleRender(ESQLQueryRender): mappings: ElasticSearchMappings = elasticsearch_mappings mitre: MitreConfig = MitreConfig() + or_token = "or" + field_value_render = ESQLRuleFieldValueRender(or_token=or_token) + def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): return [] From 3bad2f0e4cfd396ecd640090d0ee60fd34ab2acf Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 23 Jul 2024 09:07:22 +0200 Subject: [PATCH 28/40] merge prod --- uncoder-core/app/translator/core/const.py | 10 +- .../app/translator/core/exceptions/core.py | 9 +- uncoder-core/app/translator/core/mapping.py | 40 +++++- .../app/translator/core/mixins/logic.py | 18 +-- .../app/translator/core/mixins/operator.py | 2 +- .../app/translator/core/models/field.py | 136 ------------------ .../translator/core/models/query_container.py | 6 +- .../core/models/query_tokens/__init__.py | 0 .../core/models/query_tokens/field.py | 39 +++++ .../core/models/query_tokens/field_field.py | 18 +++ .../core/models/query_tokens/field_value.py | 35 +++++ .../models/query_tokens/function_value.py | 14 ++ .../models/{ => query_tokens}/identifier.py | 0 .../core/models/query_tokens/keyword.py | 21 +++ .../core/models/query_tokens/value.py | 30 ++++ uncoder-core/app/translator/core/parser.py | 55 ++++--- uncoder-core/app/translator/core/render.py | 131 ++++++++++------- uncoder-core/app/translator/core/tokenizer.py | 60 ++++++-- .../platforms/palo_alto_cortex/default.yml | 15 +- .../platforms/palo_alto_cortex/webserver.yml | 1 + .../palo_alto_cortex/windows_image_load.yml | 1 + .../mappings/platforms/qradar/default.yml | 26 +++- .../qradar/linux_process_creation.yml | 1 + .../mappings/platforms/qradar/proxy.yml | 14 +- .../mappings/platforms/qradar/webserver.yml | 43 ++++-- .../platforms/qradar/windows_image_load.yml | 3 +- .../qradar/windows_process_creation.yml | 6 +- .../platforms/qradar/windows_security.yml | 1 + .../mappings/utils/load_from_files.py | 4 +- .../app/translator/platforms/athena/const.py | 2 +- .../translator/platforms/athena/mapping.py | 3 +- .../platforms/athena/parsers/athena.py | 8 +- .../platforms/athena/renders/athena.py | 10 +- .../platforms/athena/renders/athena_cti.py | 4 +- .../translator/platforms/base/aql/const.py | 11 +- .../translator/platforms/base/aql/mapping.py | 3 - .../platforms/base/aql/parsers/aql.py | 16 +-- .../platforms/base/aql/renders/aql.py | 4 +- .../platforms/base/aql/tokenizer.py | 61 +++++++- .../platforms/base/lucene/parsers/lucene.py | 9 +- .../platforms/base/lucene/tokenizer.py | 8 +- .../platforms/base/spl/parsers/spl.py | 10 +- .../platforms/base/spl/tokenizer.py | 7 +- .../platforms/base/sql/parsers/sql.py | 9 +- .../platforms/base/sql/tokenizer.py | 4 +- .../translator/platforms/chronicle/mapping.py | 6 +- .../platforms/chronicle/parsers/chronicle.py | 13 +- .../chronicle/parsers/chronicle_rule.py | 4 +- .../platforms/chronicle/renders/chronicle.py | 6 +- .../chronicle/renders/chronicle_rule.py | 9 +- .../platforms/chronicle/tokenizer.py | 4 +- .../platforms/crowdstrike/mapping.py | 3 +- .../crowdstrike/parsers/crowdstrike.py | 4 +- .../crowdstrike/renders/crowdstrike.py | 4 +- .../platforms/elasticsearch/mapping.py | 20 ++- .../elasticsearch/parsers/elasticsearch.py | 5 +- .../elasticsearch/renders/detection_rule.py | 7 +- .../elasticsearch/renders/elast_alert.py | 7 +- .../elasticsearch/renders/elasticsearch.py | 5 +- .../platforms/elasticsearch/renders/kibana.py | 7 +- .../elasticsearch/renders/xpack_watcher.py | 7 +- .../platforms/forti_siem/mapping.py | 3 +- .../forti_siem/renders/forti_siem_rule.py | 84 ++++++----- .../app/translator/platforms/graylog/const.py | 2 +- .../translator/platforms/graylog/mapping.py | 8 +- .../platforms/graylog/parsers/graylog.py | 9 +- .../platforms/graylog/renders/graylog.py | 11 +- .../app/translator/platforms/hunters/const.py | 2 +- .../translator/platforms/hunters/mapping.py | 3 +- .../platforms/hunters/renders/hunters.py | 10 +- .../platforms/logrhythm_axon/mapping.py | 10 +- .../renders/logrhythm_axon_query.py | 65 ++++----- .../renders/logrhythm_axon_rule.py | 6 +- .../translator/platforms/logscale/mapping.py | 4 +- .../platforms/logscale/parsers/logscale.py | 14 +- .../logscale/parsers/logscale_alert.py | 2 + .../platforms/logscale/renders/logscale.py | 23 +-- .../logscale/renders/logscale_alert.py | 4 + .../platforms/logscale/tokenizer.py | 8 +- .../translator/platforms/microsoft/const.py | 2 +- .../translator/platforms/microsoft/mapping.py | 18 ++- .../microsoft/parsers/microsoft_defender.py | 8 +- .../microsoft/parsers/microsoft_sentinel.py | 14 +- .../parsers/microsoft_sentinel_rule.py | 2 + .../microsoft/renders/microsoft_defender.py | 12 +- .../renders/microsoft_defender_cti.py | 4 +- .../microsoft/renders/microsoft_sentinel.py | 4 +- .../renders/microsoft_sentinel_rule.py | 4 + .../platforms/opensearch/mapping.py | 9 +- .../opensearch/parsers/opensearch.py | 5 +- .../opensearch/renders/opensearch.py | 5 +- .../opensearch/renders/opensearch_rule.py | 15 +- .../translator/platforms/palo_alto/mapping.py | 5 +- .../palo_alto/renders/cortex_xsiam.py | 12 +- .../translator/platforms/qradar/mapping.py | 4 + .../platforms/qradar/parsers/qradar.py | 3 + .../platforms/qradar/renders/qradar.py | 3 + .../app/translator/platforms/sigma/const.py | 4 + .../app/translator/platforms/sigma/mapping.py | 3 +- .../platforms/sigma/models/compiler.py | 5 +- .../platforms/sigma/models/modifiers.py | 4 +- .../platforms/sigma/parsers/sigma.py | 11 +- .../platforms/sigma/renders/sigma.py | 20 +-- .../translator/platforms/sigma/tokenizer.py | 5 +- .../translator/platforms/splunk/mapping.py | 4 +- .../platforms/splunk/parsers/splunk.py | 7 +- .../platforms/splunk/parsers/splunk_alert.py | 2 + .../platforms/splunk/renders/splunk.py | 6 +- .../platforms/splunk/renders/splunk_alert.py | 5 + 109 files changed, 904 insertions(+), 588 deletions(-) delete mode 100644 uncoder-core/app/translator/core/models/field.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/__init__.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/field.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/field_field.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/field_value.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/function_value.py rename uncoder-core/app/translator/core/models/{ => query_tokens}/identifier.py (100%) create mode 100644 uncoder-core/app/translator/core/models/query_tokens/keyword.py create mode 100644 uncoder-core/app/translator/core/models/query_tokens/value.py create mode 100644 uncoder-core/app/translator/platforms/qradar/mapping.py diff --git a/uncoder-core/app/translator/core/const.py b/uncoder-core/app/translator/core/const.py index a8788ada..c8fd16ce 100644 --- a/uncoder-core/app/translator/core/const.py +++ b/uncoder-core/app/translator/core/const.py @@ -1,6 +1,10 @@ from typing import Union -from app.translator.core.models.field import Alias, Field, FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field import Alias, Field +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword -TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias] +QUERY_TOKEN_TYPE = Union[FieldField, FieldValue, FunctionValue, Keyword, Identifier, Field, Alias] diff --git a/uncoder-core/app/translator/core/exceptions/core.py b/uncoder-core/app/translator/core/exceptions/core.py index 47810576..8a5256e6 100644 --- a/uncoder-core/app/translator/core/exceptions/core.py +++ b/uncoder-core/app/translator/core/exceptions/core.py @@ -10,19 +10,14 @@ class BasePlatformException(BaseException): class StrictPlatformException(BasePlatformException): - field_name: str = None - - def __init__( - self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None - ): + def __init__(self, platform_name: str, fields: list[str], mapping: Optional[str] = None): message = ( f"Platform {platform_name} has strict mapping. " - f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping." + f"Source fields: {', '.join(fields)} have no mapping." f" Mapping file: {mapping}." if mapping else "" ) - self.field_name = field_name super().__init__(message) diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index bdab5f6d..78bf8b9f 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -1,10 +1,16 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Optional, TypeVar +from typing import TYPE_CHECKING, Optional, TypeVar +from app.translator.core.exceptions.core import StrictPlatformException +from app.translator.core.models.platform_details import PlatformDetails from app.translator.mappings.utils.load_from_files import LoaderFileMappings +if TYPE_CHECKING: + from app.translator.core.models.query_tokens.field import Field + + DEFAULT_MAPPING_NAME = "default" @@ -85,12 +91,16 @@ def __init__( class BasePlatformMappings: + details: PlatformDetails = None + + is_strict_mapping: bool = False skip_load_default_mappings: bool = True extend_default_mapping_with_all_fields: bool = False - def __init__(self, platform_dir: str): + def __init__(self, platform_dir: str, platform_details: PlatformDetails): self._loader = LoaderFileMappings() self._platform_dir = platform_dir + self.details = platform_details self._source_mappings = self.prepare_mapping() def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None: @@ -148,6 +158,32 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]: def default_mapping(self) -> SourceMapping: return self._source_mappings[DEFAULT_MAPPING_NAME] + def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]: + unmapped = [] + for field in field_tokens: + generic_field_name = field.get_generic_field_name(source_mapping.source_id) + mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name) + if not mapped_field and field.source_name not in unmapped: + unmapped.append(field.source_name) + + if self.is_strict_mapping and unmapped: + raise StrictPlatformException( + platform_name=self.details.name, fields=unmapped, mapping=source_mapping.source_id + ) + + return unmapped + + @staticmethod + def map_field(field: Field, source_mapping: SourceMapping) -> list[str]: + generic_field_name = field.get_generic_field_name(source_mapping.source_id) + # field can be mapped to corresponding platform field name or list of platform field names + mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name) + + if isinstance(mapped_field, str): + mapped_field = [mapped_field] + + return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name] + class BaseCommonPlatformMappings(ABC, BasePlatformMappings): def prepare_mapping(self) -> dict[str, SourceMapping]: diff --git a/uncoder-core/app/translator/core/mixins/logic.py b/uncoder-core/app/translator/core/mixins/logic.py index b24a1c99..7002e847 100644 --- a/uncoder-core/app/translator/core/mixins/logic.py +++ b/uncoder-core/app/translator/core/mixins/logic.py @@ -1,19 +1,21 @@ -from typing import Union - +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword class ANDLogicOperatorMixin: @staticmethod - def get_missed_and_token_indices(tokens: list[Union[FieldValue, Keyword, Identifier]]) -> list[int]: + def get_missed_and_token_indices(tokens: list[QUERY_TOKEN_TYPE]) -> list[int]: missed_and_indices = [] for index in range(len(tokens) - 1): token = tokens[index] next_token = tokens[index + 1] if ( - isinstance(token, (FieldValue, Keyword)) + isinstance(token, (FieldField, FieldValue, FunctionValue, Keyword)) or isinstance(token, Identifier) and token.token_type == GroupType.R_PAREN ) and not ( @@ -23,9 +25,7 @@ def get_missed_and_token_indices(tokens: list[Union[FieldValue, Keyword, Identif missed_and_indices.append(index + 1) return list(reversed(missed_and_indices)) - def add_and_token_if_missed( - self, tokens: list[Union[FieldValue, Keyword, Identifier]] - ) -> list[Union[FieldValue, Keyword, Identifier]]: + def add_and_token_if_missed(self, tokens: list[QUERY_TOKEN_TYPE]) -> list[QUERY_TOKEN_TYPE]: indices = self.get_missed_and_token_indices(tokens=tokens) for index in indices: tokens.insert(index, Identifier(token_type=LogicalOperatorType.AND)) diff --git a/uncoder-core/app/translator/core/mixins/operator.py b/uncoder-core/app/translator/core/mixins/operator.py index dee82395..dec9e3f4 100644 --- a/uncoder-core/app/translator/core/mixins/operator.py +++ b/uncoder-core/app/translator/core/mixins/operator.py @@ -19,7 +19,7 @@ from typing import Optional, Union from app.translator.core.custom_types.tokens import OperatorType -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.identifier import Identifier class WildCardMixin: diff --git a/uncoder-core/app/translator/core/models/field.py b/uncoder-core/app/translator/core/models/field.py deleted file mode 100644 index d9facb77..00000000 --- a/uncoder-core/app/translator/core/models/field.py +++ /dev/null @@ -1,136 +0,0 @@ -from typing import Optional, Union - -from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS, OperatorType -from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping -from app.translator.core.models.identifier import Identifier -from app.translator.core.str_value_manager import StrValue - - -class Alias: - def __init__(self, name: str): - self.name = name - - -class Field: - def __init__(self, source_name: str): - self.source_name = source_name - self.__generic_names_map = {} - - def get_generic_field_name(self, source_id: str) -> Optional[str]: - return self.__generic_names_map.get(source_id) - - def add_generic_names_map(self, generic_names_map: dict) -> None: - self.__generic_names_map = generic_names_map - - def set_generic_names_map(self, source_mappings: list[SourceMapping], default_mapping: SourceMapping) -> None: - generic_names_map = { - source_mapping.source_id: source_mapping.fields_mapping.get_generic_field_name(self.source_name) - or self.source_name - for source_mapping in source_mappings - } - if DEFAULT_MAPPING_NAME not in generic_names_map: - fields_mapping = default_mapping.fields_mapping - generic_names_map[DEFAULT_MAPPING_NAME] = ( - fields_mapping.get_generic_field_name(self.source_name) or self.source_name - ) - - self.__generic_names_map = generic_names_map - - -class PredefinedField: - def __init__(self, name: str): - self.name = name - - -class FieldField: - def __init__( - self, - source_name_left: str, - operator: Identifier, - source_name_right: str, - is_alias_left: bool = False, - is_alias_right: bool = False, - ): - self.field_left = Field(source_name=source_name_left) if not is_alias_left else None - self.alias_left = Alias(name=source_name_left) if is_alias_left else None - self.operator = operator - self.field_right = Field(source_name=source_name_right) if not is_alias_right else None - self.alias_right = Alias(name=source_name_right) if is_alias_right else None - - -class FieldValue: - def __init__( - self, - source_name: str, - operator: Identifier, - value: Union[int, str, StrValue, list, tuple], - is_alias: bool = False, - is_predefined_field: bool = False, - ): - # mapped by platform fields mapping - self.field = Field(source_name=source_name) if not (is_alias or is_predefined_field) else None - # not mapped - self.alias = Alias(name=source_name) if is_alias else None - # mapped by platform predefined fields mapping - self.predefined_field = PredefinedField(name=source_name) if is_predefined_field else None - - self.operator = operator - self.values = [] - self.__add_value(value) - - @property - def value(self) -> Union[int, str, StrValue, list[Union[int, str, StrValue]]]: - if isinstance(self.values, list) and len(self.values) == 1: - return self.values[0] - return self.values - - @value.setter - def value(self, new_value: Union[int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: - self.values = [] - self.__add_value(new_value) - - def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -> None: - if value and isinstance(value, (list, tuple)): - for v in value: - self.__add_value(v) - elif ( - value - and isinstance(value, str) - and value.isnumeric() - and self.operator.token_type not in STR_SEARCH_OPERATORS - ): - self.values.append(int(value)) - elif value is not None and isinstance(value, (int, str)): - self.values.append(value) - - def __repr__(self): - if self.alias: - return f"{self.alias.name} {self.operator.token_type} {self.values}" - - if self.predefined_field: - return f"{self.predefined_field.name} {self.operator.token_type} {self.values}" - - return f"{self.field.source_name} {self.operator.token_type} {self.values}" - - -class Keyword: - def __init__(self, value: Union[str, list[str]]): - self.operator: Identifier = Identifier(token_type=OperatorType.KEYWORD) - self.name = "keyword" - self.values = [] - self.__add_value(value=value) - - @property - def value(self) -> Union[str, list[str]]: - if isinstance(self.values, list) and len(self.values) == 1: - return self.values[0] - return self.values - - def __add_value(self, value: Union[str, list[str]]) -> None: - if value and isinstance(value, (list, tuple)): - self.values.extend(value) - elif value and isinstance(value, str): - self.values.append(value) - - def __repr__(self): - return f"{self.name} {self.operator.token_type} {self.values}" diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index 0d90f237..7c56c71a 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -3,11 +3,11 @@ from datetime import datetime from typing import Optional -from app.translator.core.const import TOKEN_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import DEFAULT_MAPPING_NAME -from app.translator.core.models.field import Field from app.translator.core.models.functions.base import ParsedFunctions +from app.translator.core.models.query_tokens.field import Field class MetaInfoContainer: @@ -65,6 +65,6 @@ class RawQueryDictContainer: @dataclass class TokenizedQueryContainer: - tokens: list[TOKEN_TYPE] + tokens: list[QUERY_TOKEN_TYPE] meta_info: MetaInfoContainer functions: ParsedFunctions = field(default_factory=ParsedFunctions) diff --git a/uncoder-core/app/translator/core/models/query_tokens/__init__.py b/uncoder-core/app/translator/core/models/query_tokens/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uncoder-core/app/translator/core/models/query_tokens/field.py b/uncoder-core/app/translator/core/models/query_tokens/field.py new file mode 100644 index 00000000..84d07e4e --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/field.py @@ -0,0 +1,39 @@ +from typing import Optional + +from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping + + +class Alias: + def __init__(self, name: str): + self.name = name + + +class Field: + def __init__(self, source_name: str): + self.source_name = source_name + self.__generic_names_map = {} + + def get_generic_field_name(self, source_id: str) -> Optional[str]: + return self.__generic_names_map.get(source_id) + + def add_generic_names_map(self, generic_names_map: dict) -> None: + self.__generic_names_map = generic_names_map + + def set_generic_names_map(self, source_mappings: list[SourceMapping], default_mapping: SourceMapping) -> None: + generic_names_map = { + source_mapping.source_id: source_mapping.fields_mapping.get_generic_field_name(self.source_name) + or self.source_name + for source_mapping in source_mappings + } + if DEFAULT_MAPPING_NAME not in generic_names_map: + fields_mapping = default_mapping.fields_mapping + generic_names_map[DEFAULT_MAPPING_NAME] = ( + fields_mapping.get_generic_field_name(self.source_name) or self.source_name + ) + + self.__generic_names_map = generic_names_map + + +class PredefinedField: + def __init__(self, name: str): + self.name = name diff --git a/uncoder-core/app/translator/core/models/query_tokens/field_field.py b/uncoder-core/app/translator/core/models/query_tokens/field_field.py new file mode 100644 index 00000000..86099f08 --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/field_field.py @@ -0,0 +1,18 @@ +from app.translator.core.models.query_tokens.field import Alias, Field +from app.translator.core.models.query_tokens.identifier import Identifier + + +class FieldField: + def __init__( + self, + source_name_left: str, + operator: Identifier, + source_name_right: str, + is_alias_left: bool = False, + is_alias_right: bool = False, + ): + self.field_left = Field(source_name=source_name_left) if not is_alias_left else None + self.alias_left = Alias(name=source_name_left) if is_alias_left else None + self.operator = operator + self.field_right = Field(source_name=source_name_right) if not is_alias_right else None + self.alias_right = Alias(name=source_name_right) if is_alias_right else None diff --git a/uncoder-core/app/translator/core/models/query_tokens/field_value.py b/uncoder-core/app/translator/core/models/query_tokens/field_value.py new file mode 100644 index 00000000..cf491da4 --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/field_value.py @@ -0,0 +1,35 @@ +from typing import Union + +from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS +from app.translator.core.models.query_tokens.field import Alias, Field, PredefinedField +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.value import Value +from app.translator.core.str_value_manager import StrValue + + +class FieldValue(Value): + def __init__( + self, + source_name: str, + operator: Identifier, + value: Union[bool, int, str, StrValue, list, tuple], + is_alias: bool = False, + is_predefined_field: bool = False, + ): + super().__init__(value, cast_to_int=operator.token_type not in STR_SEARCH_OPERATORS) + # mapped by platform fields mapping + self.field = Field(source_name=source_name) if not (is_alias or is_predefined_field) else None + # not mapped + self.alias = Alias(name=source_name) if is_alias else None + # mapped by platform predefined fields mapping + self.predefined_field = PredefinedField(name=source_name) if is_predefined_field else None + self.operator = operator + + def __repr__(self): + if self.alias: + return f"{self.alias.name} {self.operator.token_type} {self.values}" + + if self.predefined_field: + return f"{self.predefined_field.name} {self.operator.token_type} {self.values}" + + return f"{self.field.source_name} {self.operator.token_type} {self.values}" diff --git a/uncoder-core/app/translator/core/models/query_tokens/function_value.py b/uncoder-core/app/translator/core/models/query_tokens/function_value.py new file mode 100644 index 00000000..6ffd49bc --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/function_value.py @@ -0,0 +1,14 @@ +from typing import Union + +from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS +from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.value import Value +from app.translator.core.str_value_manager import StrValue + + +class FunctionValue(Value): + def __init__(self, function: Function, operator: Identifier, value: Union[int, str, StrValue, list, tuple]): + super().__init__(value, cast_to_int=operator.token_type not in STR_SEARCH_OPERATORS) + self.function = function + self.operator = operator diff --git a/uncoder-core/app/translator/core/models/identifier.py b/uncoder-core/app/translator/core/models/query_tokens/identifier.py similarity index 100% rename from uncoder-core/app/translator/core/models/identifier.py rename to uncoder-core/app/translator/core/models/query_tokens/identifier.py diff --git a/uncoder-core/app/translator/core/models/query_tokens/keyword.py b/uncoder-core/app/translator/core/models/query_tokens/keyword.py new file mode 100644 index 00000000..09382791 --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/keyword.py @@ -0,0 +1,21 @@ +from typing import Union + +from app.translator.core.custom_types.tokens import OperatorType +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.value import Value + + +class Keyword(Value): + def __init__(self, value: Union[str, list[str]]): + super().__init__(value) + self.operator: Identifier = Identifier(token_type=OperatorType.KEYWORD) + self.name = "keyword" + + def _add_value(self, value: Union[str, list[str]]) -> None: + if value and isinstance(value, (list, tuple)): + self.values.extend(value) + elif value and isinstance(value, str): + self.values.append(value) + + def __repr__(self): + return f"{self.name} {self.operator.token_type} {self.values}" diff --git a/uncoder-core/app/translator/core/models/query_tokens/value.py b/uncoder-core/app/translator/core/models/query_tokens/value.py new file mode 100644 index 00000000..d3d77eb0 --- /dev/null +++ b/uncoder-core/app/translator/core/models/query_tokens/value.py @@ -0,0 +1,30 @@ +from typing import Optional, Union + +from app.translator.core.str_value_manager import StrValue + + +class Value: + def __init__(self, value: Union[bool, int, str, StrValue, list, tuple], cast_to_int: bool = False): + self.values = [] + self.__cast_to_int = cast_to_int + self._add_value(value) + + @property + def value(self) -> Union[bool, int, str, StrValue, list[Union[int, str, StrValue]]]: + if isinstance(self.values, list) and len(self.values) == 1: + return self.values[0] + return self.values + + @value.setter + def value(self, new_value: Union[bool, int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: + self.values = [] + self._add_value(new_value) + + def _add_value(self, value: Optional[Union[bool, int, str, StrValue, list, tuple]]) -> None: + if value and isinstance(value, (list, tuple)): + for v in value: + self._add_value(v) + elif value and isinstance(value, str) and value.isnumeric() and self.__cast_to_int: + self.values.append(int(value)) + elif value is not None and isinstance(value, (bool, int, str)): + self.values.append(value) diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 18b50739..fcefeb69 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -18,17 +18,19 @@ import re from abc import ABC, abstractmethod -from typing import Union +from typing import Optional, Union -from app.translator.core.const import TOKEN_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword -from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.functions.base import Function from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.models.query_tokens.field import Field +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue from app.translator.core.tokenizer import QueryTokenizer @@ -55,24 +57,35 @@ class PlatformQueryParser(QueryParser, ABC): tokenizer: QueryTokenizer = None platform_functions: PlatformFunctions = None - def get_fields_tokens(self, tokens: list[Union[FieldValue, Keyword, Identifier]]) -> list[Field]: - return [token.field for token in self.tokenizer.filter_tokens(tokens, FieldValue)] - - def get_tokens_and_source_mappings( - self, query: str, log_sources: dict[str, Union[str, list[str]]] - ) -> tuple[list[TOKEN_TYPE], list[SourceMapping]]: + def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]: if not query: raise TokenizerGeneralException("Can't translate empty query. Please provide more details") - tokens = self.tokenizer.tokenize(query=query) - field_tokens = self.get_fields_tokens(tokens=tokens) + return self.tokenizer.tokenize(query=query) + + def get_field_tokens( + self, query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None + ) -> list[Field]: + field_tokens = [] + for token in query_tokens: + if isinstance(token, FieldValue): + field_tokens.append(token.field) + elif isinstance(token, FieldField): + if token.field_left: + field_tokens.append(token.field_left) + if token.field_right: + field_tokens.append(token.field_right) + elif isinstance(token, FunctionValue): + field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args([token.function])) + + if functions: + field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args(functions)) + + return field_tokens + + def get_source_mappings( + self, field_tokens: list[Field], log_sources: dict[str, Union[str, list[str]]] + ) -> list[SourceMapping]: field_names = [field.source_name for field in field_tokens] source_mappings = self.mappings.get_suitable_source_mappings(field_names=field_names, **log_sources) self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping) - - return tokens, source_mappings - - def set_functions_fields_generic_names( - self, functions: ParsedFunctions, source_mappings: list[SourceMapping] - ) -> None: - field_tokens = self.tokenizer.get_field_tokens_from_func_args(args=functions.functions) - self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping) + return source_mappings diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index b4b1ccc5..6158b679 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -22,7 +22,7 @@ from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.const import TOKEN_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType @@ -31,11 +31,15 @@ from app.translator.core.exceptions.parser import UnsupportedOperatorException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping -from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword, PredefinedField from app.translator.core.models.functions.base import Function, RenderedFunctions -from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer +from app.translator.core.models.query_tokens.field import Field, PredefinedField +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.str_value_manager import StrValue, StrValueManager @@ -74,6 +78,10 @@ def _get_value_type(field_name: str, value: Union[int, str, StrValue], value_typ def _wrap_str_value(value: str) -> str: return value + @staticmethod + def _map_bool_value(value: bool) -> str: + return "true" if value else "false" + def _pre_process_value( self, field: str, value: Union[int, str, StrValue], value_type: str = ValueType.value, wrap_str: bool = False ) -> Union[int, str]: @@ -84,6 +92,8 @@ def _pre_process_value( if isinstance(value, str): value = self.str_value_manager.escape_manager.escape(value, value_type) return self._wrap_str_value(value) if wrap_str else value + if isinstance(value, bool): + return self._map_bool_value(value) return value def _pre_process_values_list( @@ -174,6 +184,7 @@ class QueryRender(ABC): details: PlatformDetails = None is_single_line_comment: bool = False unsupported_functions_text = "Unsupported functions were excluded from the result query:" + unmapped_fields_text = "Unmapped fields: " platform_functions: PlatformFunctions = None @@ -196,6 +207,11 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions: return query + def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[str]]) -> str: + if fields: + return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{', '.join(fields)}") + return query + def wrap_with_comment(self, value: str) -> str: return f"{self.comment_symbol} {value}" @@ -206,7 +222,6 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta class PlatformQueryRender(QueryRender): mappings: BasePlatformMappings = None - is_strict_mapping: bool = False or_token = "or" and_token = "and" @@ -237,35 +252,23 @@ def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], fu def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions: return self.platform_functions.render(functions, source_mapping) - def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]: - generic_field_name = field.get_generic_field_name(source_mapping.source_id) - # field can be mapped to corresponding platform field name or list of platform field names - mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name) - if not mapped_field and self.is_strict_mapping: - raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name) - - if isinstance(mapped_field, str): - mapped_field = [mapped_field] - - return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name] - def map_predefined_field(self, predefined_field: PredefinedField) -> str: if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)): - if self.is_strict_mapping: - raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name) + if self.mappings.is_strict_mapping: + raise StrictPlatformException(platform_name=self.details.name, fields=[predefined_field.name]) return predefined_field.name return mapped_predefined_field_name - def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): if token.alias: mapped_fields = [token.alias.name] elif token.predefined_field: mapped_fields = [self.map_predefined_field(token.predefined_field)] else: - mapped_fields = self.map_field(token.field, source_mapping) + mapped_fields = self.mappings.map_field(token.field, source_mapping) joined = self.logical_operators_map[LogicalOperatorType.OR].join( [ self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) @@ -275,9 +278,13 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp return self.group_token % joined if len(mapped_fields) > 1 else joined if isinstance(token, FieldField): alias_left, field_left = token.alias_left, token.field_left - mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping) + mapped_fields_left = ( + [alias_left.name] if alias_left else self.mappings.map_field(field_left, source_mapping) + ) alias_right, field_right = token.alias_right, token.field_right - mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping) + mapped_fields_right = ( + [alias_right.name] if alias_right else self.mappings.map_field(field_right, source_mapping) + ) cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right)) joined = self.logical_operators_map[LogicalOperatorType.OR].join( [ @@ -286,9 +293,12 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp ] ) return self.group_token % joined if len(cross_paired_fields) > 1 else joined - if isinstance(token, Function): - func_render = self.platform_functions.manager.get_in_query_render(token.name) - return func_render.render(token, source_mapping) + if isinstance(token, FunctionValue): + func_render = self.platform_functions.manager.get_render(token.function.name) + rendered_func = func_render.render(token.function, source_mapping) + return self.field_value_render.apply_field_value( + field=rendered_func, operator=token.operator, value=token.value + ) if isinstance(token, Keyword): return self.field_value_render.apply_field_value(field="", operator=token.operator, value=token.value) if token.token_type in LogicalOperatorType: @@ -296,16 +306,11 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp return token.token_type - def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str: + def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str: result_values = [] - unmapped_fields = set() for token in tokens: - try: - result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) - except StrictPlatformException as err: - unmapped_fields.add(err.field_name) - if unmapped_fields: - raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields)) + result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) + return "".join(result_values) def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str: @@ -338,11 +343,13 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: query = self._join_query_parts(prefix, query, functions) query = self.wrap_with_meta_info(query, meta_info) + query = self.wrap_with_unmapped_fields(query, unmapped_fields) return self.wrap_with_not_supported_functions(query, not_supported_functions) @staticmethod @@ -404,45 +411,59 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap mapped_field = source_mapping.fields_mapping.get_platform_field_name( generic_field_name=generic_field_name ) - if not mapped_field and self.is_strict_mapping: - raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name) + if not mapped_field and self.mappings.is_strict_mapping: + raise StrictPlatformException( + platform_name=self.details.name, fields=[field.source_name], mapping=source_mapping.source_id + ) if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping): for prefix in prefix_list: if prefix not in defined_raw_log_fields: defined_raw_log_fields.append(prefix) return "\n".join(defined_raw_log_fields) + def _generate_from_tokenized_query_container_by_source_mapping( + self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping + ) -> str: + unmapped_fields = self.mappings.check_fields_mapping_existence( + query_container.meta_info.query_fields, source_mapping + ) + rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) + prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix) + + if source_mapping.raw_log_fields: + defined_raw_log_fields = self.generate_raw_log_fields( + fields=query_container.meta_info.query_fields, source_mapping=source_mapping + ) + prefix += f"\n{defined_raw_log_fields}" + query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) + not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported + return self.finalize_query( + prefix=prefix, + query=query, + functions=rendered_functions.rendered, + not_supported_functions=not_supported_functions, + unmapped_fields=unmapped_fields, + meta_info=query_container.meta_info, + source_mapping=source_mapping, + ) + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} errors = [] source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) for source_mapping in source_mappings: - rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) - prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix) try: - if source_mapping.raw_log_fields: - defined_raw_log_fields = self.generate_raw_log_fields( - fields=query_container.meta_info.query_fields, source_mapping=source_mapping - ) - prefix += f"\n{defined_raw_log_fields}" - result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) - except StrictPlatformException as err: - errors.append(err) - continue - else: - not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported - finalized_query = self.finalize_query( - prefix=prefix, - query=result, - functions=rendered_functions.rendered, - not_supported_functions=not_supported_functions, - meta_info=query_container.meta_info, - source_mapping=source_mapping, + finalized_query = self._generate_from_tokenized_query_container_by_source_mapping( + query_container, source_mapping ) if return_only_first_query_ctx_var.get() is True: return finalized_query queries_map[source_mapping.source_id] = finalized_query + except StrictPlatformException as err: + errors.append(err) + continue + if not queries_map and errors: raise errors[0] return self.finalize(queries_map) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index ff9385ba..6c04787c 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -20,7 +20,7 @@ from abc import ABC, abstractmethod from typing import Any, ClassVar, Optional, Union -from app.translator.core.const import TOKEN_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -29,8 +29,8 @@ TokenizerGeneralException, UnsupportedOperatorException, ) +from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function from app.translator.core.models.functions.eval import EvalArg from app.translator.core.models.functions.group_by import GroupByFunction @@ -38,14 +38,19 @@ from app.translator.core.models.functions.rename import RenameArg from app.translator.core.models.functions.sort import SortArg from app.translator.core.models.functions.union import UnionFunction -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field import Field +from app.translator.core.models.query_tokens.field_field import FieldField +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.str_value_manager import StrValue, StrValueManager from app.translator.tools.utils import get_match_group class BaseTokenizer(ABC): @abstractmethod - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: raise NotImplementedError @@ -58,12 +63,22 @@ class QueryTokenizer(BaseTokenizer): fields_operator_map: ClassVar[dict[str, str]] = {} operators_map: ClassVar[dict[str, str]] = {} # used to generate re pattern. so the keys order is important - logical_operator_pattern = r"^(?Pand|or|not|AND|OR|NOT)\s+" + logical_operators_map: ClassVar[dict[str, str]] = { + "and": LogicalOperatorType.AND, + "AND": LogicalOperatorType.AND, + "or": LogicalOperatorType.OR, + "OR": LogicalOperatorType.OR, + "not": LogicalOperatorType.NOT, + "NOT": LogicalOperatorType.NOT, + } + _logical_operator_pattern = f"(?P{'|'.join(logical_operators_map)})" + logical_operator_pattern = rf"^{_logical_operator_pattern}\s+" field_value_pattern = r"""^___field___\s*___operator___\s*___value___""" base_value_pattern = r"(?:___value_pattern___)" # do not modify, use subclasses to define this attribute field_pattern: str = None + function_pattern: str = None _value_pattern: str = None value_pattern: str = None multi_value_pattern: str = None @@ -73,6 +88,7 @@ class QueryTokenizer(BaseTokenizer): wildcard_symbol = None escape_manager: EscapeManager = None str_value_manager: StrValueManager = None + platform_functions: PlatformFunctions = None def __init_subclass__(cls, **kwargs): cls._validate_re_patterns() @@ -268,9 +284,16 @@ def _check_field_value_match(self, query: str, white_space_pattern: str = r"\s+" return False + def search_function_value(self, query: str) -> tuple[FunctionValue, str]: + ... + + @staticmethod + def _check_function_value_match(query: str) -> bool: # noqa: ARG004 + return False + def _get_next_token( self, query: str - ) -> tuple[Union[FieldValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]: + ) -> tuple[Union[FieldValue, FunctionValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]: query = query.strip("\n").strip(" ").strip("\n") if query.startswith(GroupType.L_PAREN): return Identifier(token_type=GroupType.L_PAREN), query[1:] @@ -280,15 +303,23 @@ def _get_next_token( logical_operator = logical_operator_search.group("logical_operator") pos = logical_operator_search.end() return Identifier(token_type=logical_operator.lower()), query[pos:] + if self.platform_functions and self._check_function_value_match(query): # noqa: SIM102 + if search_result := self.search_function_value(query): + return search_result if self._check_field_value_match(query): return self.search_field_value(query) if self.keyword_pattern and re.match(self.keyword_pattern, query): return self.search_keyword(query) - raise TokenizerGeneralException("Unsupported query entry") + unsupported_query_entry = self._get_unsupported_query_entry(query) + raise TokenizerGeneralException(f"Unsupported query entry: {unsupported_query_entry}") + + def _get_unsupported_query_entry(self, query: str) -> str: + split_by_logical_operator = re.split(rf"\s+{self._logical_operator_pattern}\s+", query, maxsplit=1) + return split_by_logical_operator[0] @staticmethod - def _validate_parentheses(tokens: list[TOKEN_TYPE]) -> None: + def _validate_parentheses(tokens: list[QUERY_TOKEN_TYPE]) -> None: parentheses = [] for token in tokens: if isinstance(token, Identifier) and token.token_type in (GroupType.L_PAREN, GroupType.R_PAREN): @@ -301,7 +332,7 @@ def _validate_parentheses(tokens: list[TOKEN_TYPE]) -> None: if parentheses: raise QueryParenthesesException - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: tokenized = [] while query: next_token, sliced_query = self._get_next_token(query=query) @@ -320,8 +351,9 @@ def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: @staticmethod def filter_tokens( - tokens: list[TOKEN_TYPE], token_type: Union[type[FieldValue], type[Field], type[Keyword], type[Identifier]] - ) -> list[TOKEN_TYPE]: + tokens: list[QUERY_TOKEN_TYPE], + token_type: Union[type[FieldValue], type[Field], type[FieldField], type[Keyword], type[Identifier]], + ) -> list[QUERY_TOKEN_TYPE]: return [token for token in tokens if isinstance(token, token_type)] def get_field_tokens_from_func_args( # noqa: PLR0912 @@ -339,11 +371,15 @@ def get_field_tokens_from_func_args( # noqa: PLR0912 elif isinstance(arg, FieldValue): if arg.field: result.append(arg.field) + elif isinstance(arg, FunctionValue): + result.extend(self.get_field_tokens_from_func_args(args=[arg.function])) elif isinstance(arg, GroupByFunction): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_])) - elif isinstance(arg, (JoinFunction, UnionFunction)): + elif isinstance(arg, JoinFunction): + result.extend(self.get_field_tokens_from_func_args(args=arg.condition)) + elif isinstance(arg, UnionFunction): continue elif isinstance(arg, Function): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml index fa904aaf..f767249b 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml @@ -14,6 +14,7 @@ field_mapping: ProcessName: - xdm.target.process.name - xdm.source.process.name + ProcessPath: xdm.target.process.executable.path ImageLoaded: - xdm.target.process.executable.filename - xdm.source.process.executable.filename @@ -46,6 +47,7 @@ field_mapping: c-uri-query: xdm.network.http.url QueryName: xdm.network.dns.dns_question.name Application: xdm.network.application_protocol + sourceNetwork: xdm.source.subnet SourceHostName: xdm.source.host.hostname DestinationHostname: xdm.target.host.hostname Hashes: @@ -64,7 +66,7 @@ field_mapping: dns-query: xdm.network.dns.dns_question.name dns-answer: xdm.network.dns.dns_resource_record.value dns-record: xdm.network.dns.dns_question.name - FileName: xdm.target.file.path + FileName: xdm.target.file.filename IpAddress: xdm.source.ipv4 IpPort: xdm.source.port LogonProcessName: xdm.target.process.executable.path @@ -126,3 +128,14 @@ field_mapping: DestinationOS: xdm.target.host.os url_category: xdm.network.http.url_category EventSeverity: xdm.alert.severity + duration: xdm.event.duration + ThreatName: xdm.alert.original_threat_id + AnalyzerName: xdm.observer.type + Classification: xdm.alert.category + ResultCode: xdm.event.outcome_reason + Technique: xdm.alert.mitre_techniques + Action: xdm.event.outcome + FileExtension: xdm.target.file.extension + Workstation: xdm.source.host.hostname + RegistryKey: xdm.target.registry.key + RegistryValue: xdm.target.registry.value diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/webserver.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/webserver.yml index 7a1eaa84..505012f0 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/webserver.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/webserver.yml @@ -15,5 +15,6 @@ field_mapping: cs-uri-stem: xdm.network.http.url cs-uri-query: xdm.network.http.url c-uri-path: xdm.network.http.url + cs-host: xdm.network.http.domain uri_path: xdm.network.http.url cs-uri: xdm.network.http.url diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_image_load.yml index 69a100ec..98e62b8f 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_image_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_image_load.yml @@ -9,6 +9,7 @@ default_log_source: field_mapping: ImageLoaded: action_module_path + FileExtension: action_file_extension md5: action_module_md5 sha256: action_module_sha256 User: actor_effective_username diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml index 00dcef55..5ff97d09 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml @@ -19,6 +19,7 @@ field_mapping: src-port: - SourcePort - localport + - sourcePort src-ip: - sourceip - source_ip @@ -34,6 +35,9 @@ field_mapping: User: - userName - EventUserName + - Alert Threat Cause Actor Name + - Username + - Security ID CommandLine: Command Protocol: - IPProtocol @@ -41,6 +45,7 @@ field_mapping: Application: - Application - application + sourceNetwork: sourceNetwork SourceHostName: - HostCount-source - identityHostName @@ -77,4 +82,23 @@ field_mapping: EventSeverity: EventSeverity Source: - Source - - source \ No newline at end of file + - source + duration: duration + ThreatName: + - Threat Name + - Alert Blocked Threat Category + AnalyzerName: Analyzer Name + Classification: Classification + ResultCode: Alert Reason Code + Technique: Technique + Action: Action + Workstation: Machine Identifier + GroupMembership: Role Name + FileName: + - Filename + - File Name + RegistryKey: + - Registry Key + - Target Object + RegistryValue: RegistryValue + ProcessPath: Process Path diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml index 8fddefd6..67e3db21 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml @@ -14,6 +14,7 @@ field_mapping: CommandLine: - Command - ASACommand + - Command Arguments Image: Process Path ParentCommandLine: Parent Command ParentImage: Parent Process Path diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml index 193bc79c..b179f8c0 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml @@ -13,18 +13,22 @@ field_mapping: - URL - XForceCategoryByURL c-useragent: User Agent - cs-method: HTTP Method + cs-method: + - HTTP Method + - Method cs-bytes: Bytes Sent #cs-cookie-vars: cs-cookie-vars c-uri-extension: URL c-uri-query: - URL - URL Path + - URL Query String #cs-cookie: cs-cookie cs-host: - UrlHost - URL Host - URL Domain + - HTTP Host cs-referrer: - URL Referrer - Referrer URL @@ -32,6 +36,10 @@ field_mapping: r-dns: - UrlHost - URL Host - sc-status: HTTP Response Code + sc-status: + - HTTP Response Code + - Response Code #post-body: post-body - url_category: XForceCategoryByURL \ No newline at end of file + url_category: + - XForceCategoryByURL + - Web Category \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/webserver.yml b/uncoder-core/app/translator/mappings/platforms/qradar/webserver.yml index 11a769f6..b43fbc8d 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/webserver.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/webserver.yml @@ -9,17 +9,34 @@ default_log_source: devicetype: 10 field_mapping: - c-uri: URL - c-useragent: c-useragent - cs-method: cs-method + c-uri: + - URL + - XForceCategoryByURL + c-useragent: User Agent + cs-method: + - HTTP Method + - Method cs-bytes: Bytes Sent - cs-cookie-vars: cs-cookie-vars - c-uri-extension: c-uri-extension - c-uri-query: URL - cs-cookie: cs-cookie - cs-host: cs-host - cs-referrer: URL Referrer - cs-version: cs-version - r-dns: r-dns - sc-status: sc-status - post-body: post-body \ No newline at end of file + #cs-cookie-vars: cs-cookie-vars + c-uri-extension: URL + c-uri-query: + - URL + - URL Path + - URL Query String + #cs-cookie: cs-cookie + cs-host: + - UrlHost + - URL Host + - URL Domain + - HTTP Host + cs-referrer: + - URL Referrer + - Referrer URL + cs-version: HTTP Version + r-dns: + - UrlHost + - URL Host + sc-status: + - HTTP Response Code + - Response Code + #post-body: post-body \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_image_load.yml index bb1189f6..79d3bd66 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_image_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_image_load.yml @@ -21,4 +21,5 @@ field_mapping: - Signature Status - SignatureStatus OriginalFileName: OriginalFileName - Signed: Signed \ No newline at end of file + Signed: Signed + FileExtension: File Extension \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml index 1886343a..fcad6da1 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml @@ -14,15 +14,19 @@ field_mapping: CommandLine: - Command - Encoded Argument + - Command Arguments CurrentDirectory: CurrentDirectory Hashes: File Hash Image: - Process Path - Process Name - DGApplication + - ProcessName IntegrityLevel: IntegrityLevel ParentCommandLine: Parent Command - ParentImage: Parent Process Path + ParentImage: + - Parent Process Path + - ParentProcessName ParentUser: ParentUser Product: Product User: diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml index 9ccb1fbe..2a4c9919 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml @@ -12,6 +12,7 @@ field_mapping: EventID: - Event ID - EventID + - qidEventId ParentImage: Parent Process Path AccessMask: AccessMask AccountName: Account Name diff --git a/uncoder-core/app/translator/mappings/utils/load_from_files.py b/uncoder-core/app/translator/mappings/utils/load_from_files.py index 6bd48a47..89ea2a94 100644 --- a/uncoder-core/app/translator/mappings/utils/load_from_files.py +++ b/uncoder-core/app/translator/mappings/utils/load_from_files.py @@ -6,6 +6,7 @@ from app.translator.const import APP_PATH COMMON_FIELD_MAPPING_FILE_NAME = "common.yml" +DEFAULT_FIELD_MAPPING_FILE_NAME = "default.yml" class LoaderFileMappings: @@ -23,8 +24,9 @@ def load_mapping(mapping_file_path: str) -> dict: def load_platform_mappings(self, platform_dir: str) -> Generator[dict, None, None]: platform_path = os.path.join(self.base_mapping_filepath, platform_dir) for mapping_file in os.listdir(platform_path): - if mapping_file != COMMON_FIELD_MAPPING_FILE_NAME: + if mapping_file not in (COMMON_FIELD_MAPPING_FILE_NAME, DEFAULT_FIELD_MAPPING_FILE_NAME): yield self.load_mapping(mapping_file_path=os.path.join(platform_path, mapping_file)) + yield self.load_mapping(mapping_file_path=os.path.join(platform_path, DEFAULT_FIELD_MAPPING_FILE_NAME)) def load_common_mapping(self, platform_dir: str) -> dict: platform_path = os.path.join(self.base_mapping_filepath, platform_dir) diff --git a/uncoder-core/app/translator/platforms/athena/const.py b/uncoder-core/app/translator/platforms/athena/const.py index 1f286117..db261b69 100644 --- a/uncoder-core/app/translator/platforms/athena/const.py +++ b/uncoder-core/app/translator/platforms/athena/const.py @@ -9,4 +9,4 @@ "alt_platform_name": "OCSF", } -athena_details = PlatformDetails(**ATHENA_QUERY_DETAILS) +athena_query_details = PlatformDetails(**ATHENA_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/athena/mapping.py b/uncoder-core/app/translator/platforms/athena/mapping.py index ab33bd7a..d15d5156 100644 --- a/uncoder-core/app/translator/platforms/athena/mapping.py +++ b/uncoder-core/app/translator/platforms/athena/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.athena.const import athena_query_details class AthenaLogSourceSignature(LogSourceSignature): @@ -40,4 +41,4 @@ def get_suitable_source_mappings(self, field_names: list[str], table: Optional[s return suitable_source_mappings -athena_mappings = AthenaMappings(platform_dir="athena") +athena_query_mappings = AthenaMappings(platform_dir="athena", platform_details=athena_query_details) diff --git a/uncoder-core/app/translator/platforms/athena/parsers/athena.py b/uncoder-core/app/translator/platforms/athena/parsers/athena.py index 565f4165..9e2bd555 100644 --- a/uncoder-core/app/translator/platforms/athena/parsers/athena.py +++ b/uncoder-core/app/translator/platforms/athena/parsers/athena.py @@ -18,14 +18,14 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager -from app.translator.platforms.athena.const import athena_details -from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings +from app.translator.platforms.athena.const import athena_query_details +from app.translator.platforms.athena.mapping import AthenaMappings, athena_query_mappings from app.translator.platforms.base.sql.parsers.sql import SqlQueryParser @parser_manager.register_supported_by_roota class AthenaQueryParser(SqlQueryParser): - details: PlatformDetails = athena_details - mappings: AthenaMappings = athena_mappings + details: PlatformDetails = athena_query_details + mappings: AthenaMappings = athena_query_mappings query_delimiter_pattern = r"\sFROM\s\S*\sWHERE\s" table_pattern = r"\sFROM\s(?P
[a-zA-Z\.\-\*]+)\sWHERE\s" diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena.py b/uncoder-core/app/translator/platforms/athena/renders/athena.py index 8550c94a..2b431af2 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena.py @@ -19,19 +19,19 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.athena.const import athena_details -from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings +from app.translator.platforms.athena.const import athena_query_details +from app.translator.platforms.athena.mapping import AthenaMappings, athena_query_mappings from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender class AthenaFieldValueRender(SqlFieldValueRender): - details: PlatformDetails = athena_details + details: PlatformDetails = athena_query_details @render_manager.register class AthenaQueryRender(SqlQueryRender): - details: PlatformDetails = athena_details - mappings: AthenaMappings = athena_mappings + details: PlatformDetails = athena_query_details + mappings: AthenaMappings = athena_query_mappings or_token = "OR" diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena_cti.py b/uncoder-core/app/translator/platforms/athena/renders/athena_cti.py index aa4f986b..c46290e8 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena_cti.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena_cti.py @@ -20,13 +20,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render_cti import RenderCTI from app.translator.managers import render_cti_manager -from app.translator.platforms.athena.const import athena_details +from app.translator.platforms.athena.const import athena_query_details from app.translator.platforms.athena.mappings.athena_cti import DEFAULT_ATHENA_MAPPING @render_cti_manager.register class AthenaCTI(RenderCTI): - details: PlatformDetails = athena_details + details: PlatformDetails = athena_query_details field_value_template: str = "{key} = '{value}'" or_operator: str = " OR " diff --git a/uncoder-core/app/translator/platforms/base/aql/const.py b/uncoder-core/app/translator/platforms/base/aql/const.py index 1df7cdd1..b2503bd4 100644 --- a/uncoder-core/app/translator/platforms/base/aql/const.py +++ b/uncoder-core/app/translator/platforms/base/aql/const.py @@ -1,7 +1,10 @@ +from app.translator.core.custom_types.values import ValueType + UTF8_PAYLOAD_PATTERN = r"UTF8\(payload\)" -NUM_VALUE_PATTERN = r"(?P\d+(?:\.\d+)*)" -SINGLE_QUOTES_VALUE_PATTERN = ( - r"""'(?P(?:[:a-zA-Zа-яА-Я\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{\}\[\]\s]|'')*)'""" # noqa: RUF001 -) +NUM_VALUE_PATTERN = rf"(?P<{ValueType.number_value}>\d+(?:\.\d+)*)" +SINGLE_QUOTES_VALUE_PATTERN = rf"""'(?P<{ValueType.single_quotes_value}>(?:[:a-zA-Zа-яА-Я\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{{\}}\[\]\s]|'')*)'""" # noqa: E501,RUF001 TABLE_PATTERN = r"\s+FROM\s+[a-zA-Z.\-*]+" TABLE_GROUP_PATTERN = r"\s+FROM\s+(?P
[a-zA-Z.\-*]+)" + +FIELD_NAME_PATTERN = rf"(?P<{ValueType.no_quotes_value}>[a-zA-Z0-9\._\-]+)" +DOUBLE_QUOTES_FIELD_NAME_PATTERN = rf'"(?P<{ValueType.double_quotes_value}>[a-zA-Z0-9\._\-\s]+)"' diff --git a/uncoder-core/app/translator/platforms/base/aql/mapping.py b/uncoder-core/app/translator/platforms/base/aql/mapping.py index c0fb4b2f..a975a1b4 100644 --- a/uncoder-core/app/translator/platforms/base/aql/mapping.py +++ b/uncoder-core/app/translator/platforms/base/aql/mapping.py @@ -90,6 +90,3 @@ def get_suitable_source_mappings( suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]] return suitable_source_mappings - - -aql_mappings = AQLMappings(platform_dir="qradar") diff --git a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py index f911ea27..8d6fc601 100644 --- a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py @@ -26,14 +26,12 @@ from app.translator.platforms.base.aql.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, TABLE_GROUP_PATTERN from app.translator.platforms.base.aql.functions import AQLFunctions, aql_functions from app.translator.platforms.base.aql.log_source_map import LOG_SOURCE_FUNCTIONS_MAP -from app.translator.platforms.base.aql.mapping import AQLMappings, aql_mappings -from app.translator.platforms.base.aql.tokenizer import AQLTokenizer, aql_tokenizer +from app.translator.platforms.base.aql.tokenizer import AQLTokenizer from app.translator.tools.utils import get_match_group class AQLQueryParser(PlatformQueryParser): - tokenizer: AQLTokenizer = aql_tokenizer - mappings: AQLMappings = aql_mappings + tokenizer: AQLTokenizer = AQLTokenizer(aql_functions) platform_functions: AQLFunctions = aql_functions log_source_functions = ("LOGSOURCENAME", "LOGSOURCEGROUPNAME") @@ -116,10 +114,10 @@ def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources, functions = self._parse_query(raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) - self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens, functions.functions) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py index 6c0c1665..58fbc3ff 100644 --- a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py @@ -23,7 +23,7 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue -from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings +from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager @@ -121,8 +121,6 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: class AQLQueryRender(PlatformQueryRender): - mappings: AQLMappings = aql_mappings - or_token = "OR" and_token = "AND" not_token = "NOT" diff --git a/uncoder-core/app/translator/platforms/base/aql/tokenizer.py b/uncoder-core/app/translator/platforms/base/aql/tokenizer.py index 54a797eb..ff04be20 100644 --- a/uncoder-core/app/translator/platforms/base/aql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/aql/tokenizer.py @@ -21,11 +21,22 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.functions import PlatformFunctions +from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.function_value import FunctionValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.str_value_manager import StrValue from app.translator.core.tokenizer import QueryTokenizer -from app.translator.platforms.base.aql.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, UTF8_PAYLOAD_PATTERN +from app.translator.platforms.base.aql.const import ( + DOUBLE_QUOTES_FIELD_NAME_PATTERN, + FIELD_NAME_PATTERN, + NUM_VALUE_PATTERN, + SINGLE_QUOTES_VALUE_PATTERN, + UTF8_PAYLOAD_PATTERN, +) +from app.translator.platforms.base.aql.functions.const import AQLFunctionGroupType from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager from app.translator.tools.utils import get_match_group @@ -46,6 +57,7 @@ class AQLTokenizer(QueryTokenizer): multi_value_operators_map: ClassVar[dict[str, str]] = {"in": OperatorType.EQ} field_pattern = r'(?P"[a-zA-Z\._\-\s]+"|[a-zA-Z\._\-]+)' + function_pattern = rf"(?P[a-zA-Z_]+)\((?:{FIELD_NAME_PATTERN}|{DOUBLE_QUOTES_FIELD_NAME_PATTERN})\)" bool_value_pattern = rf"(?P<{ValueType.bool_value}>true|false)\s*" _value_pattern = rf"{NUM_VALUE_PATTERN}|{bool_value_pattern}|{SINGLE_QUOTES_VALUE_PATTERN}" multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>[:a-zA-Z\"\*0-9=+%#\-_\/\\'\,.&^@!\(\s]*)\)""" @@ -54,6 +66,9 @@ class AQLTokenizer(QueryTokenizer): wildcard_symbol = "%" str_value_manager = aql_str_value_manager + def __init__(self, platform_functions: PlatformFunctions = None): + self.platform_functions = platform_functions + @staticmethod def should_process_value_wildcards(operator: Optional[str]) -> bool: return operator and operator.lower() in ("like", "ilike") @@ -79,7 +94,7 @@ def get_operator_and_value( return super().get_operator_and_value(match, mapped_operator, operator) def escape_field_name(self, field_name: str) -> str: - return field_name.replace('"', r"\"").replace(" ", r"\ ") + return field_name.replace('"', r"\"").replace(" ", r"\ ").replace("(", "\(").replace(")", "\)") @staticmethod def create_field_value(field_name: str, operator: Identifier, value: Union[str, list]) -> FieldValue: @@ -93,5 +108,39 @@ def search_keyword(self, query: str) -> tuple[Keyword, str]: pos = keyword_search.end() return keyword, query[pos:] - -aql_tokenizer = AQLTokenizer() + def _search_function_value(self, function: Function, query: str) -> tuple[FunctionValue, str]: + operator = self.search_operator(query, function.raw) + if self.is_multi_value_flow(function.raw, operator, query): + query, grouped_values = self.search_multi_value(query=query, operator=operator, field_name=function.raw) + tokens = [ # always consists of 1 element + FunctionValue(function=function, operator=Identifier(token_type=op), value=values) + for op, values in grouped_values.items() + ] + return tokens[0], query + + query, operator, value = self.search_single_value(query=query, operator=operator, field_name=function.raw) + operator_token = Identifier(token_type=operator) + return FunctionValue(function=function, operator=operator_token, value=value), query + + def search_function_value(self, query: str) -> tuple[FunctionValue, str]: + str_conversion_func_parser = self.platform_functions.manager.get_parser(AQLFunctionGroupType.str_conversion) + if str_conversion_func_parser and (func_match := str_conversion_func_parser.get_func_match(query)): + function = str_conversion_func_parser.parse(func_match.name, func_match.match) + return self._search_function_value(function, query) + + return super().search_function_value(query) + + def _check_function_value_match(self, query: str, white_space_pattern: str = r"\s+") -> bool: + single_value_operator_group = rf"(?:{'|'.join(self.single_value_operators_map)})" + single_value_pattern = rf"""{self.function_pattern}\s*{single_value_operator_group}\s*{self.value_pattern}\s*""" + if re.match(single_value_pattern, query, re.IGNORECASE): + return True + + if self.multi_value_operators_map: + multi_value_operator_group = rf"(?:{'|'.join(self.multi_value_operators_map)})" + pattern = f"{self.function_pattern}{white_space_pattern}{multi_value_operator_group}{white_space_pattern}" + multi_value_pattern = rf"{pattern}{self.multi_value_pattern}" + if re.match(multi_value_pattern, query, re.IGNORECASE): + return True + + return False diff --git a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py index c748c1e4..5fb57284 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py @@ -47,9 +47,10 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources = self._parse_query(raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py b/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py index eb54b7ea..b56f5bee 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py @@ -19,11 +19,13 @@ import re from typing import ClassVar, Optional, Union +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.mixins.logic import ANDLogicOperatorMixin -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.str_value_manager import StrValue from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.base.lucene.escape_manager import lucene_escape_manager @@ -135,6 +137,6 @@ def _check_field_value_match(self, query: str, white_space_pattern: str = r"\s*" return super()._check_field_value_match(query, white_space_pattern=white_space_pattern) - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: tokens = super().tokenize(query=query) return self.add_and_token_if_missed(tokens=tokens) diff --git a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py index 92ba415d..27a1559d 100644 --- a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py @@ -67,10 +67,10 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain return self.platform_functions.parse_tstats_func(raw_query_container) query, log_sources, functions = self._parse_query(raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) - self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens, functions.functions) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/base/spl/tokenizer.py b/uncoder-core/app/translator/platforms/base/spl/tokenizer.py index 8a030519..57a5a695 100644 --- a/uncoder-core/app/translator/platforms/base/spl/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/spl/tokenizer.py @@ -17,13 +17,12 @@ """ import re -from typing import Any, ClassVar, Optional, Union +from typing import Any, ClassVar, Optional +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.mixins.logic import ANDLogicOperatorMixin -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.base.spl.const import DOUBLE_QUOTES_VALUE_PATTERN as D_Q_V_PATTERN from app.translator.platforms.base.spl.const import FIELD_PATTERN @@ -77,6 +76,6 @@ def get_operator_and_value( return super().get_operator_and_value(match, mapped_operator, operator) - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: tokens = super().tokenize(query=query) return self.add_and_token_if_missed(tokens=tokens) diff --git a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py index d324d4ba..4a882467 100644 --- a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py @@ -43,9 +43,10 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, Optional[str]]]: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources = self._parse_query(raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py index 944d3c9b..8292ca14 100644 --- a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py @@ -21,8 +21,8 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType -from app.translator.core.models.field import FieldValue -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer from app.translator.tools.utils import get_match_group diff --git a/uncoder-core/app/translator/platforms/chronicle/mapping.py b/uncoder-core/app/translator/platforms/chronicle/mapping.py index bea60c0e..d341eef8 100644 --- a/uncoder-core/app/translator/platforms/chronicle/mapping.py +++ b/uncoder-core/app/translator/platforms/chronicle/mapping.py @@ -1,4 +1,5 @@ from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.chronicle.const import chronicle_query_details, chronicle_rule_details class ChronicleLogSourceSignature(LogSourceSignature): @@ -10,6 +11,8 @@ def __str__(self) -> str: class ChronicleMappings(BasePlatformMappings): + is_strict_mapping = True + def prepare_log_source_signature(self, mapping: dict) -> ChronicleLogSourceSignature: ... @@ -28,4 +31,5 @@ def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMap return suitable_source_mappings -chronicle_mappings = ChronicleMappings(platform_dir="chronicle") +chronicle_query_mappings = ChronicleMappings(platform_dir="chronicle", platform_details=chronicle_query_details) +chronicle_rule_mappings = ChronicleMappings(platform_dir="chronicle", platform_details=chronicle_rule_details) diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py index 8c0e8431..7c50cb06 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py @@ -21,22 +21,23 @@ from app.translator.core.parser import PlatformQueryParser from app.translator.managers import parser_manager from app.translator.platforms.chronicle.const import chronicle_query_details -from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings +from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_query_mappings from app.translator.platforms.chronicle.tokenizer import ChronicleQueryTokenizer @parser_manager.register_supported_by_roota class ChronicleQueryParser(PlatformQueryParser): - mappings: ChronicleMappings = chronicle_mappings + mappings: ChronicleMappings = chronicle_query_mappings tokenizer: ChronicleQueryTokenizer = ChronicleQueryTokenizer() details: PlatformDetails = chronicle_query_details wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - tokens, source_mappings = self.get_tokens_and_source_mappings(raw_query_container.query, {}) - fields_tokens = self.get_fields_tokens(tokens=tokens) + query_tokens = self.get_query_tokens(raw_query_container.query) + field_tokens = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(field_tokens, {}) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py index c7929714..888b55eb 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py @@ -23,7 +23,7 @@ from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager from app.translator.platforms.chronicle.const import chronicle_rule_details -from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings +from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_rule_mappings from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser from app.translator.platforms.chronicle.tokenizer import ChronicleRuleTokenizer @@ -35,7 +35,7 @@ class ChronicleRuleParser(ChronicleQueryParser): meta_info_pattern = "meta:\n(?P[a-zA-Z0-9_\\\.*,>–<—~#$’`:;%+^\|?!@\s\"/=\-&'\(\)\[\]]+)\n\s+events:" # noqa: RUF001 rule_pattern = "events:\n\s*(?P[a-zA-Z\w0-9_%{}\|\.,!#^><:~\s\"\/=+?\-–&;$()`\*@\[\]'\\\]+)\n\s+condition:" # noqa: RUF001 event_name_pattern = "condition:\n\s*(?P\$[a-zA-Z_0-9]+)\n" - mappings: ChronicleMappings = chronicle_mappings + mappings: ChronicleMappings = chronicle_rule_mappings tokenizer = ChronicleRuleTokenizer() def __parse_rule(self, rule: str) -> tuple[str, str, str]: diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py index 8bcbe56f..7642929f 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py @@ -27,7 +27,7 @@ from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import chronicle_query_details from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager -from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings +from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_query_mappings class ChronicleFieldValueRender(BaseFieldValueRender): @@ -101,9 +101,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 @render_manager.register class ChronicleQueryRender(PlatformQueryRender): details: PlatformDetails = chronicle_query_details - mappings: ChronicleMappings = chronicle_mappings - - is_strict_mapping = True + mappings: ChronicleMappings = chronicle_query_mappings or_token = "or" and_token = "and" diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py index 1961c72b..3f59f42b 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py @@ -26,6 +26,7 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details +from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_rule_mappings from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValueRender, ChronicleQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated Chronicle Security rule." @@ -84,6 +85,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @render_manager.register class ChronicleSecurityRuleRender(ChronicleQueryRender): details: PlatformDetails = chronicle_rule_details + mappings: ChronicleMappings = chronicle_rule_mappings or_token = "or" field_value_render = ChronicleRuleFieldValueRender(or_token=or_token) @@ -108,7 +110,8 @@ def finalize_query( functions: str, meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 - not_supported_functions: Optional[list] = None, # noqa: ARG002, + not_supported_functions: Optional[list] = None, # , + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -124,4 +127,6 @@ def finalize_query( rule = rule.replace("", meta_info.status) rule = rule.replace("", ", ".join(meta_info.false_positives)) rule = rule.replace("", ", ".join(meta_info.tags)) - return rule.replace("", str(meta_info.date)) + rule = rule.replace("", str(meta_info.date)) + rule = self.wrap_with_unmapped_fields(rule, unmapped_fields) + return self.wrap_with_not_supported_functions(rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/chronicle/tokenizer.py b/uncoder-core/app/translator/platforms/chronicle/tokenizer.py index 5278da4a..a0943952 100644 --- a/uncoder-core/app/translator/platforms/chronicle/tokenizer.py +++ b/uncoder-core/app/translator/platforms/chronicle/tokenizer.py @@ -21,8 +21,8 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType -from app.translator.core.models.field import FieldValue -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager from app.translator.tools.utils import get_match_group diff --git a/uncoder-core/app/translator/platforms/crowdstrike/mapping.py b/uncoder-core/app/translator/platforms/crowdstrike/mapping.py index 80de46a3..5c41399b 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/mapping.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.crowdstrike.const import crowdstrike_query_details class CrowdStrikeLogSourceSignature(LogSourceSignature): @@ -38,4 +39,4 @@ def get_suitable_source_mappings(self, field_names: list[str], event_simpleName: return suitable_source_mappings or [self._source_mappings[DEFAULT_MAPPING_NAME]] -crowdstrike_mappings = CrowdstrikeMappings(platform_dir="crowdstrike") +crowdstrike_query_mappings = CrowdstrikeMappings(platform_dir="crowdstrike", platform_details=crowdstrike_query_details) diff --git a/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py index 80130636..08ec0b7f 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/parsers/crowdstrike.py @@ -21,7 +21,7 @@ from app.translator.platforms.base.spl.parsers.spl import SplQueryParser from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions -from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings +from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_query_mappings @parser_manager.register_supported_by_roota @@ -31,7 +31,7 @@ class CrowdStrikeQueryParser(SplQueryParser): log_source_pattern = r"___source_type___\s*=\s*(?:\"(?P[%a-zA-Z_*:0-9\-/]+)\"|(?P[%a-zA-Z_*:0-9\-/]+))(?:\s+(?:and|or)\s+|\s+)?" # noqa: E501 log_source_key_types = ("event_simpleName",) - mappings: CrowdstrikeMappings = crowdstrike_mappings + mappings: CrowdstrikeMappings = crowdstrike_query_mappings platform_functions: CrowdStrikeFunctions = crowd_strike_functions wrapped_with_comment_pattern = r"^\s*`(?:|\n|.)*`" diff --git a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py index 3e5900cc..40911708 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py @@ -22,7 +22,7 @@ from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions -from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings +from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_query_mappings class CrowdStrikeFieldValueRender(SplFieldValueRender): @@ -32,7 +32,7 @@ class CrowdStrikeFieldValueRender(SplFieldValueRender): @render_manager.register class CrowdStrikeQueryRender(SplQueryRender): details: PlatformDetails = crowdstrike_query_details - mappings: CrowdstrikeMappings = crowdstrike_mappings + mappings: CrowdstrikeMappings = crowdstrike_query_mappings platform_functions: CrowdStrikeFunctions = None or_token = "OR" diff --git a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py index 6c71ab29..b0489fbf 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py @@ -1,8 +1,16 @@ from app.translator.platforms.base.lucene.mapping import LuceneMappings +from app.translator.platforms.elasticsearch.const import ( + elastalert_details, + elasticsearch_lucene_query_details, + elasticsearch_rule_details, + kibana_rule_details, + xpack_watcher_details, +) - -class ElasticSearchMappings(LuceneMappings): - pass - - -elasticsearch_mappings = ElasticSearchMappings(platform_dir="elasticsearch") +elasticsearch_lucene_query_mappings = LuceneMappings( + platform_dir="elasticsearch", platform_details=elasticsearch_lucene_query_details +) +elasticsearch_rule_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=elasticsearch_rule_details) +elastalert_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=elastalert_details) +kibana_rule_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=kibana_rule_details) +xpack_watcher_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=xpack_watcher_details) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch.py index a3bad851..2f287a93 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch.py @@ -18,12 +18,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.parsers.lucene import LuceneQueryParser from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import elasticsearch_lucene_query_mappings @parser_manager.register_supported_by_roota class ElasticSearchQueryParser(LuceneQueryParser): details: PlatformDetails = elasticsearch_lucene_query_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = elasticsearch_lucene_query_mappings diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py index 0b7b20c4..6904e47b 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -26,8 +26,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_DETECTION_RULE, elasticsearch_rule_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import elasticsearch_rule_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( ElasticSearchFieldValue, ElasticSearchQueryRender, @@ -43,7 +44,7 @@ class ElasticSearchRuleFieldValue(ElasticSearchFieldValue): @render_manager.register class ElasticSearchRuleRender(ElasticSearchQueryRender): details: PlatformDetails = elasticsearch_rule_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = elasticsearch_rule_mappings mitre: MitreConfig = MitreConfig() or_token = "OR" @@ -86,6 +87,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -109,4 +111,5 @@ def finalize_query( } ) rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py index 9d7914ab..6b28a9e3 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py @@ -24,8 +24,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_ALERT, elastalert_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import elastalert_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( ElasticSearchFieldValue, ElasticSearchQueryRender, @@ -43,7 +44,7 @@ class ElasticAlertRuleFieldValue(ElasticSearchFieldValue): @render_manager.register class ElastAlertRuleRender(ElasticSearchQueryRender): details: PlatformDetails = elastalert_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = elastalert_mappings or_token = "OR" and_token = "AND" @@ -59,6 +60,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -75,4 +77,5 @@ def finalize_query( ) rule = rule.replace("", meta_info.title or _AUTOGENERATED_TEMPLATE) rule = rule.replace("", _SEVERITIES_MAP[meta_info.severity]) + rule = self.wrap_with_unmapped_fields(rule, unmapped_fields) return self.wrap_with_not_supported_functions(rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py index 2e6a12f0..817707ae 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py @@ -19,9 +19,10 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import elasticsearch_lucene_query_mappings class ElasticSearchFieldValue(LuceneFieldValueRender): @@ -31,7 +32,7 @@ class ElasticSearchFieldValue(LuceneFieldValueRender): @render_manager.register class ElasticSearchQueryRender(LuceneQueryRender): details: PlatformDetails = elasticsearch_lucene_query_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = elasticsearch_lucene_query_mappings or_token = "OR" field_value_render = ElasticSearchFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py index 53a4acf5..e799bdfe 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py @@ -25,8 +25,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import KIBANA_RULE, KIBANA_SEARCH_SOURCE_JSON, kibana_rule_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import kibana_rule_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( ElasticSearchFieldValue, ElasticSearchQueryRender, @@ -43,7 +44,7 @@ class KibanaFieldValue(ElasticSearchFieldValue): @render_manager.register class KibanaRuleRender(ElasticSearchQueryRender): details: PlatformDetails = kibana_rule_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = kibana_rule_mappings or_token = "OR" field_value_render = KibanaFieldValue(or_token=or_token) @@ -55,6 +56,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -74,4 +76,5 @@ def finalize_query( references=meta_info.references, ) rule_str = json.dumps(rule, indent=4, sort_keys=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py index d8421977..eab58aa4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py @@ -25,8 +25,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import XPACK_WATCHER_RULE, xpack_watcher_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import xpack_watcher_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( ElasticSearchFieldValue, ElasticSearchQueryRender, @@ -43,7 +44,7 @@ class XpackWatcherRuleFieldValue(ElasticSearchFieldValue): @render_manager.register class XPackWatcherRuleRender(ElasticSearchQueryRender): details: PlatformDetails = xpack_watcher_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = xpack_watcher_mappings or_token = "OR" field_value_render = XpackWatcherRuleFieldValue(or_token=or_token) @@ -55,6 +56,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -78,4 +80,5 @@ def finalize_query( rule["input"]["search"]["request"]["indices"] = indices rule["actions"]["send_email"]["email"]["subject"] = meta_info.title or _AUTOGENERATED_TEMPLATE rule_str = json.dumps(rule, indent=4, sort_keys=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/forti_siem/mapping.py b/uncoder-core/app/translator/platforms/forti_siem/mapping.py index 64c9f075..4fed2dbe 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/mapping.py +++ b/uncoder-core/app/translator/platforms/forti_siem/mapping.py @@ -6,6 +6,7 @@ LogSourceSignature, SourceMapping, ) +from app.translator.platforms.forti_siem.const import forti_siem_rule_details class FortiSiemLogSourceSignature(LogSourceSignature): @@ -57,4 +58,4 @@ def get_suitable_source_mappings(self, field_names: list[str], event_type: Optio return suitable_source_mappings -forti_siem_mappings = FortiSiemMappings(platform_dir="forti_siem") +forti_siem_rule_mappings = FortiSiemMappings(platform_dir="forti_siem", platform_details=forti_siem_rule_details) diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index dfbc2ee6..18a4976e 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -18,17 +18,16 @@ from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.const import TOKEN_TYPE -from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import FieldValue -from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager @@ -37,7 +36,7 @@ SOURCES_EVENT_TYPES_CONTAINERS_MAP, forti_siem_rule_details, ) -from app.translator.platforms.forti_siem.mapping import FortiSiemMappings, forti_siem_mappings +from app.translator.platforms.forti_siem.mapping import FortiSiemMappings, forti_siem_rule_mappings from app.translator.platforms.forti_siem.str_value_manager import forti_siem_str_value_manager from app.translator.tools.utils import concatenate_str @@ -186,7 +185,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 @render_manager.register class FortiSiemRuleRender(PlatformQueryRender): details: PlatformDetails = forti_siem_rule_details - mappings: FortiSiemMappings = forti_siem_mappings + mappings: FortiSiemMappings = forti_siem_rule_mappings or_token = "OR" and_token = "AND" @@ -197,7 +196,7 @@ class FortiSiemRuleRender(PlatformQueryRender): field_value_render = FortiSiemFieldValueRender(or_token=or_token) @staticmethod - def __is_negated_token(prev_token: TOKEN_TYPE) -> bool: + def __is_negated_token(prev_token: QUERY_TOKEN_TYPE) -> bool: return isinstance(prev_token, Identifier) and prev_token.token_type == LogicalOperatorType.NOT @staticmethod @@ -208,7 +207,7 @@ def __should_negate(is_negated_token: bool = False, negation_ctx: bool = False) return is_negated_token or negation_ctx @staticmethod - def __negate_token(token: TOKEN_TYPE) -> None: + def __negate_token(token: QUERY_TOKEN_TYPE) -> None: if isinstance(token, Identifier): if token.token_type == LogicalOperatorType.AND: token.token_type = LogicalOperatorType.OR @@ -218,7 +217,7 @@ def __negate_token(token: TOKEN_TYPE) -> None: token_type = token.operator.token_type token.operator.token_type = _NOT_OPERATORS_MAP_SUBSTITUTES.get(token_type) or token_type - def __replace_not_tokens(self, tokens: list[TOKEN_TYPE]) -> list[TOKEN_TYPE]: + def __replace_not_tokens(self, tokens: list[QUERY_TOKEN_TYPE]) -> list[QUERY_TOKEN_TYPE]: not_token_indices = [] negation_ctx_stack = [] for index, token in enumerate(tokens[1:], start=1): @@ -244,40 +243,37 @@ def __replace_not_tokens(self, tokens: list[TOKEN_TYPE]) -> list[TOKEN_TYPE]: return tokens - def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: - queries_map = {} - source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) - - for source_mapping in source_mappings: - is_event_type_set = False - field_values = [token for token in query_container.tokens if isinstance(token, FieldValue)] - mapped_fields_set = set() - for field_value in field_values: - mapped_fields = self.map_field(field_value.field, source_mapping) - mapped_fields_set = mapped_fields_set.union(set(mapped_fields)) - if _EVENT_TYPE_FIELD in mapped_fields: - is_event_type_set = True - self.__update_event_type_values(field_value, source_mapping.source_id) - - tokens = self.__replace_not_tokens(query_container.tokens) - result = self.generate_query(tokens=tokens, source_mapping=source_mapping) - prefix = "" if is_event_type_set else self.generate_prefix(source_mapping.log_source_signature) - rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) - not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported - finalized_query = self.finalize_query( - prefix=prefix, - query=result, - functions=rendered_functions.rendered, - not_supported_functions=not_supported_functions, - meta_info=query_container.meta_info, - source_mapping=source_mapping, - fields=mapped_fields_set, - ) - if return_only_first_query_ctx_var.get() is True: - return finalized_query - queries_map[source_mapping.source_id] = finalized_query - - return self.finalize(queries_map) + def _generate_from_tokenized_query_container_by_source_mapping( + self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping + ) -> str: + unmapped_fields = self.mappings.check_fields_mapping_existence( + query_container.meta_info.query_fields, source_mapping + ) + is_event_type_set = False + field_values = [token for token in query_container.tokens if isinstance(token, FieldValue)] + mapped_fields_set = set() + for field_value in field_values: + mapped_fields = self.mappings.map_field(field_value.field, source_mapping) + mapped_fields_set = mapped_fields_set.union(set(mapped_fields)) + if _EVENT_TYPE_FIELD in mapped_fields: + is_event_type_set = True + self.__update_event_type_values(field_value, source_mapping.source_id) + + tokens = self.__replace_not_tokens(query_container.tokens) + result = self.generate_query(tokens=tokens, source_mapping=source_mapping) + prefix = "" if is_event_type_set else self.generate_prefix(source_mapping.log_source_signature) + rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) + not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported + return self.finalize_query( + prefix=prefix, + query=result, + functions=rendered_functions.rendered, + not_supported_functions=not_supported_functions, + unmapped_fields=unmapped_fields, + meta_info=query_container.meta_info, + source_mapping=source_mapping, + fields=mapped_fields_set, + ) @staticmethod def __update_event_type_values(field_value: FieldValue, source_id: str) -> None: @@ -307,6 +303,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, fields: Optional[set[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 @@ -324,6 +321,7 @@ def finalize_query( rule = rule.replace("", query) rule = rule.replace("", ", ".join(args_list)) rule = rule.replace("", self.get_attr_str(fields.copy())) + rule = self.wrap_with_unmapped_fields(rule, unmapped_fields) return self.wrap_with_not_supported_functions(rule, not_supported_functions) @staticmethod diff --git a/uncoder-core/app/translator/platforms/graylog/const.py b/uncoder-core/app/translator/platforms/graylog/const.py index c68bfda6..f13757f5 100644 --- a/uncoder-core/app/translator/platforms/graylog/const.py +++ b/uncoder-core/app/translator/platforms/graylog/const.py @@ -9,4 +9,4 @@ } -graylog_details = PlatformDetails(**GRAYLOG_QUERY_DETAILS) +graylog_query_details = PlatformDetails(**GRAYLOG_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/graylog/mapping.py b/uncoder-core/app/translator/platforms/graylog/mapping.py index 12e95bb3..42edc609 100644 --- a/uncoder-core/app/translator/platforms/graylog/mapping.py +++ b/uncoder-core/app/translator/platforms/graylog/mapping.py @@ -1,8 +1,4 @@ from app.translator.platforms.base.lucene.mapping import LuceneMappings +from app.translator.platforms.graylog.const import graylog_query_details - -class GraylogMappings(LuceneMappings): - pass - - -graylog_mappings = GraylogMappings(platform_dir="graylog") +graylog_query_mappings = LuceneMappings(platform_dir="graylog", platform_details=graylog_query_details) diff --git a/uncoder-core/app/translator/platforms/graylog/parsers/graylog.py b/uncoder-core/app/translator/platforms/graylog/parsers/graylog.py index a4707a09..6252cd66 100644 --- a/uncoder-core/app/translator/platforms/graylog/parsers/graylog.py +++ b/uncoder-core/app/translator/platforms/graylog/parsers/graylog.py @@ -18,12 +18,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.parsers.lucene import LuceneQueryParser -from app.translator.platforms.graylog.const import graylog_details -from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings +from app.translator.platforms.graylog.const import graylog_query_details +from app.translator.platforms.graylog.mapping import graylog_query_mappings @parser_manager.register_supported_by_roota class GraylogQueryParser(LuceneQueryParser): - details: PlatformDetails = graylog_details - mappings: GraylogMappings = graylog_mappings + details: PlatformDetails = graylog_query_details + mappings: LuceneMappings = graylog_query_mappings diff --git a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py index 986ddd93..77be5c30 100644 --- a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py +++ b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py @@ -19,19 +19,20 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender -from app.translator.platforms.graylog.const import graylog_details -from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings +from app.translator.platforms.graylog.const import graylog_query_details +from app.translator.platforms.graylog.mapping import graylog_query_mappings class GraylogFieldValue(LuceneFieldValueRender): - details: PlatformDetails = graylog_details + details: PlatformDetails = graylog_query_details @render_manager.register class GraylogQueryRender(LuceneQueryRender): - details: PlatformDetails = graylog_details - mappings: GraylogMappings = graylog_mappings + details: PlatformDetails = graylog_query_details + mappings: LuceneMappings = graylog_query_mappings or_token = "OR" field_value_render = GraylogFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/hunters/const.py b/uncoder-core/app/translator/platforms/hunters/const.py index fbeff6a1..eb61c622 100644 --- a/uncoder-core/app/translator/platforms/hunters/const.py +++ b/uncoder-core/app/translator/platforms/hunters/const.py @@ -8,4 +8,4 @@ "group_id": "hunters", } -hunters_details = PlatformDetails(**HUNTERS_QUERY_DETAILS) +hunters_query_details = PlatformDetails(**HUNTERS_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/hunters/mapping.py b/uncoder-core/app/translator/platforms/hunters/mapping.py index 28f37e28..a7236eec 100644 --- a/uncoder-core/app/translator/platforms/hunters/mapping.py +++ b/uncoder-core/app/translator/platforms/hunters/mapping.py @@ -1,4 +1,5 @@ from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.hunters.const import hunters_query_details class HuntersLogSourceSignature(LogSourceSignature): @@ -32,4 +33,4 @@ def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMap return suitable_source_mappings -hunters_mappings = HuntersMappings(platform_dir="hunters") +hunters_query_mappings = HuntersMappings(platform_dir="hunters", platform_details=hunters_query_details) diff --git a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py index 3c73c234..4e977a16 100644 --- a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py +++ b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py @@ -20,18 +20,18 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender -from app.translator.platforms.hunters.const import hunters_details -from app.translator.platforms.hunters.mapping import HuntersMappings, hunters_mappings +from app.translator.platforms.hunters.const import hunters_query_details +from app.translator.platforms.hunters.mapping import HuntersMappings, hunters_query_mappings class HuntersFieldValueRender(SqlFieldValueRender): - details: PlatformDetails = hunters_details + details: PlatformDetails = hunters_query_details @render_manager.register class HuntersQueryRender(SqlQueryRender): - details: PlatformDetails = hunters_details - mappings: HuntersMappings = hunters_mappings + details: PlatformDetails = hunters_query_details + mappings: HuntersMappings = hunters_query_mappings or_token = "OR" diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py b/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py index 477d5e29..f034c40f 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.logrhythm_axon.const import logrhythm_axon_query_details, logrhythm_axon_rule_details class LogRhythmAxonLogSourceSignature(LogSourceSignature): @@ -15,6 +16,8 @@ def __str__(self) -> str: class LogRhythmAxonMappings(BasePlatformMappings): + is_strict_mapping = True + def prepare_mapping(self) -> dict[str, SourceMapping]: source_mappings = {} for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): @@ -44,4 +47,9 @@ def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMap return suitable_source_mappings -logrhythm_axon_mappings = LogRhythmAxonMappings(platform_dir="logrhythm_axon") +logrhythm_axon_query_mappings = LogRhythmAxonMappings( + platform_dir="logrhythm_axon", platform_details=logrhythm_axon_query_details +) +logrhythm_axon_rule_mappings = LogRhythmAxonMappings( + platform_dir="logrhythm_axon", platform_details=logrhythm_axon_rule_details +) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 95dbc40a..b81f5453 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -20,21 +20,20 @@ from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import LogicalOperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.core import StrictPlatformException from app.translator.core.exceptions.render import BaseRenderException from app.translator.core.mapping import LogSourceSignature, SourceMapping -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_axon_query_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_query_escape_manager -from app.translator.platforms.logrhythm_axon.mapping import LogRhythmAxonMappings, logrhythm_axon_mappings +from app.translator.platforms.logrhythm_axon.mapping import LogRhythmAxonMappings, logrhythm_axon_query_mappings class LogRhythmRegexRenderException(BaseRenderException): @@ -206,10 +205,9 @@ class LogRhythmAxonQueryRender(PlatformQueryRender): field_value_render = LogRhythmAxonFieldValueRender(or_token=or_token) - mappings: LogRhythmAxonMappings = logrhythm_axon_mappings + mappings: LogRhythmAxonMappings = logrhythm_axon_query_mappings comment_symbol = "//" is_single_line_comment = True - is_strict_mapping = True @staticmethod def _finalize_search_query(query: str) -> str: @@ -218,10 +216,10 @@ def _finalize_search_query(query: str) -> str: def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 return str(log_source_signature) - def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue) and token.field: try: - mapped_fields = self.map_field(token.field, source_mapping) + mapped_fields = self.mappings.map_field(token.field, source_mapping) except StrictPlatformException: try: return self.field_value_render.apply_field_value( @@ -242,30 +240,27 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp return super().apply_token(token, source_mapping) - def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: - queries_map = {} - source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) - - for source_mapping in source_mappings: - prefix = self.generate_prefix(source_mapping.log_source_signature) - if "product" in query_container.meta_info.parsed_logsources: - prefix = f"{prefix} CONTAINS {query_container.meta_info.parsed_logsources['product'][0]}" - else: - prefix = f"{prefix} CONTAINS anything" - - result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) - rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) - not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported - finalized_query = self.finalize_query( - prefix=prefix, - query=result, - functions=rendered_functions.rendered, - not_supported_functions=not_supported_functions, - meta_info=query_container.meta_info, - source_mapping=source_mapping, - ) - if return_only_first_query_ctx_var.get() is True: - return finalized_query - queries_map[source_mapping.source_id] = finalized_query - - return self.finalize(queries_map) + def _generate_from_tokenized_query_container_by_source_mapping( + self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping + ) -> str: + unmapped_fields = self.mappings.check_fields_mapping_existence( + query_container.meta_info.query_fields, source_mapping + ) + prefix = self.generate_prefix(source_mapping.log_source_signature) + if "product" in query_container.meta_info.parsed_logsources: + prefix = f"{prefix} CONTAINS {query_container.meta_info.parsed_logsources['product'][0]}" + else: + prefix = f"{prefix} CONTAINS anything" + + result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) + rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) + not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported + return self.finalize_query( + prefix=prefix, + query=result, + functions=rendered_functions.rendered, + not_supported_functions=not_supported_functions, + unmapped_fields=unmapped_fields, + meta_info=query_container.meta_info, + source_mapping=source_mapping, + ) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py index 2e68c2d1..614df7d2 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py @@ -28,6 +28,7 @@ from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager +from app.translator.platforms.logrhythm_axon.mapping import LogRhythmAxonMappings, logrhythm_axon_rule_mappings from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import ( LogRhythmAxonFieldValueRender, LogRhythmAxonQueryRender, @@ -52,6 +53,7 @@ class LogRhythmAxonRuleFieldValueRender(LogRhythmAxonFieldValueRender): @render_manager.register class LogRhythmAxonRuleRender(LogRhythmAxonQueryRender): details: PlatformDetails = logrhythm_axon_rule_details + mappings: LogRhythmAxonMappings = logrhythm_axon_rule_mappings or_token = "or" field_value_render = LogRhythmAxonRuleFieldValueRender(or_token=or_token) @@ -63,6 +65,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -89,8 +92,9 @@ def finalize_query( ) if meta_info.output_table_fields: rule["observationPipeline"]["pattern"]["operations"][0]["logObserved"]["groupByFields"] = [ - self.map_field(field, source_mapping)[0] for field in meta_info.output_table_fields + self.mappings.map_field(field, source_mapping)[0] for field in meta_info.output_table_fields ] json_rule = json.dumps(rule, indent=4, sort_keys=False) + json_rule = self.wrap_with_unmapped_fields(json_rule, unmapped_fields) return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/mapping.py b/uncoder-core/app/translator/platforms/logscale/mapping.py index 3856cba8..a3e9004e 100644 --- a/uncoder-core/app/translator/platforms/logscale/mapping.py +++ b/uncoder-core/app/translator/platforms/logscale/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.logscale.const import logscale_alert_details, logscale_query_details class LogScaleLogSourceSignature(LogSourceSignature): @@ -34,4 +35,5 @@ def get_suitable_source_mappings(self, field_names: list[str]) -> list[SourceMap return suitable_source_mappings -logscale_mappings = LogScaleMappings(platform_dir="logscale") +logscale_query_mappings = LogScaleMappings(platform_dir="logscale", platform_details=logscale_query_details) +logscale_alert_mappings = LogScaleMappings(platform_dir="logscale", platform_details=logscale_alert_details) diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py index e1015ff2..4f6fb9d9 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py @@ -23,7 +23,7 @@ from app.translator.managers import parser_manager from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.functions import LogScaleFunctions, log_scale_functions -from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings +from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_query_mappings from app.translator.platforms.logscale.tokenizer import LogScaleTokenizer @@ -32,7 +32,7 @@ class LogScaleQueryParser(PlatformQueryParser): details: PlatformDetails = logscale_query_details platform_functions: LogScaleFunctions = log_scale_functions tokenizer = LogScaleTokenizer() - mappings: LogScaleMappings = logscale_mappings + mappings: LogScaleMappings = logscale_query_mappings wrapped_with_comment_pattern = r"^\s*/\*(?:|\n|.)*\*/" @@ -42,10 +42,10 @@ def _parse_query(self, query: str) -> tuple[str, ParsedFunctions]: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, functions = self._parse_query(query=raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, {}) - fields_tokens = self.get_fields_tokens(tokens=tokens) - self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens, functions.functions) + source_mappings = self.get_source_mappings(field_tokens, {}) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py index a9cbd603..d4935a4e 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale_alert.py @@ -21,12 +21,14 @@ from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager from app.translator.platforms.logscale.const import logscale_alert_details +from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_alert_mappings from app.translator.platforms.logscale.parsers.logscale import LogScaleQueryParser @parser_manager.register class LogScaleAlertParser(LogScaleQueryParser, JsonRuleMixin): details: PlatformDetails = logscale_alert_details + mappings: LogScaleMappings = logscale_alert_mappings def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: rule = self.load_rule(text=text) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py index e1ed4818..1ca23243 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py @@ -17,18 +17,16 @@ ----------------------------------------------------------------- """ -from typing import Optional, Union +from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.mapping import SourceMapping from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.models.query_container import MetaInfoContainer from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.escape_manager import logscale_escape_manager from app.translator.platforms.logscale.functions import LogScaleFunctions, log_scale_functions -from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings +from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_query_mappings class LogScaleFieldValueRender(BaseFieldValueRender): @@ -95,7 +93,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @render_manager.register class LogScaleQueryRender(PlatformQueryRender): details: PlatformDetails = logscale_query_details - mappings: LogScaleMappings = logscale_mappings + mappings: LogScaleMappings = logscale_query_mappings platform_functions: LogScaleFunctions = None or_token = "or" @@ -110,18 +108,3 @@ def init_platform_functions(self) -> None: def wrap_with_comment(self, value: str) -> str: return f"/* {value} */" - - def finalize_query( - self, - prefix: str, - query: str, - functions: str, - meta_info: Optional[MetaInfoContainer] = None, - source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 - not_supported_functions: Optional[list] = None, - *args, # noqa: ARG002 - **kwargs, # noqa: ARG002 - ) -> str: - query = super().finalize_query(prefix=prefix, query=query, functions=functions) - query = self.wrap_with_meta_info(query, meta_info) - return self.wrap_with_not_supported_functions(query, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py index a6628045..57fe1edf 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py @@ -26,6 +26,7 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details +from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_alert_mappings from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValueRender, LogScaleQueryRender from app.translator.tools.utils import get_rule_description_str @@ -39,6 +40,7 @@ class LogScaleAlertFieldValueRender(LogScaleFieldValueRender): @render_manager.register class LogScaleAlertRender(LogScaleQueryRender): details: PlatformDetails = logscale_alert_details + mappings: LogScaleMappings = logscale_alert_mappings or_token = "or" field_value_render = LogScaleAlertFieldValueRender(or_token=or_token) @@ -50,6 +52,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -71,4 +74,5 @@ def finalize_query( ) rule_str = json.dumps(rule, indent=4, sort_keys=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/tokenizer.py b/uncoder-core/app/translator/platforms/logscale/tokenizer.py index c765c8a9..9c7c33e5 100644 --- a/uncoder-core/app/translator/platforms/logscale/tokenizer.py +++ b/uncoder-core/app/translator/platforms/logscale/tokenizer.py @@ -17,13 +17,13 @@ """ import re -from typing import Any, ClassVar, Optional, Union +from typing import Any, ClassVar, Optional +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.mixins.logic import ANDLogicOperatorMixin -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.logscale.escape_manager import logscale_escape_manager from app.translator.tools.utils import get_match_group @@ -71,6 +71,6 @@ def _get_next_token(self, query: str) -> (list, str): return super()._get_next_token(query) - def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]: + def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: tokens = super().tokenize(query=query) return self.add_and_token_if_missed(tokens=tokens) diff --git a/uncoder-core/app/translator/platforms/microsoft/const.py b/uncoder-core/app/translator/platforms/microsoft/const.py index 44dcf698..02a2a7d0 100644 --- a/uncoder-core/app/translator/platforms/microsoft/const.py +++ b/uncoder-core/app/translator/platforms/microsoft/const.py @@ -42,6 +42,6 @@ "group_id": "microsoft-defender", } -microsoft_defender_details = PlatformDetails(**MICROSOFT_DEFENDER_DETAILS) +microsoft_defender_query_details = PlatformDetails(**MICROSOFT_DEFENDER_DETAILS) microsoft_sentinel_query_details = PlatformDetails(**MICROSOFT_SENTINEL_QUERY_DETAILS) microsoft_sentinel_rule_details = PlatformDetails(**MICROSOFT_SENTINEL_RULE_DETAILS) diff --git a/uncoder-core/app/translator/platforms/microsoft/mapping.py b/uncoder-core/app/translator/platforms/microsoft/mapping.py index 0c32b522..4add9858 100644 --- a/uncoder-core/app/translator/platforms/microsoft/mapping.py +++ b/uncoder-core/app/translator/platforms/microsoft/mapping.py @@ -1,6 +1,11 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.microsoft.const import ( + microsoft_defender_query_details, + microsoft_sentinel_query_details, + microsoft_sentinel_rule_details, +) class MicrosoftSentinelLogSourceSignature(LogSourceSignature): @@ -37,7 +42,12 @@ def get_suitable_source_mappings(self, field_names: list[str], table: list[str]) return suitable_source_mappings -microsoft_sentinel_mappings = MicrosoftSentinelMappings(platform_dir="microsoft_sentinel") +microsoft_sentinel_query_mappings = MicrosoftSentinelMappings( + platform_dir="microsoft_sentinel", platform_details=microsoft_sentinel_query_details +) +microsoft_sentinel_rule_mappings = MicrosoftSentinelMappings( + platform_dir="microsoft_sentinel", platform_details=microsoft_sentinel_rule_details +) class MicrosoftDefenderLogSourceSignature(MicrosoftSentinelLogSourceSignature): @@ -45,10 +55,14 @@ class MicrosoftDefenderLogSourceSignature(MicrosoftSentinelLogSourceSignature): class MicrosoftDefenderMappings(MicrosoftSentinelMappings): + is_strict_mapping = True + def prepare_log_source_signature(self, mapping: dict) -> MicrosoftDefenderLogSourceSignature: tables = mapping.get("log_source", {}).get("table") default_log_source = mapping["default_log_source"] return MicrosoftDefenderLogSourceSignature(tables=tables, default_source=default_log_source) -microsoft_defender_mappings = MicrosoftDefenderMappings(platform_dir="microsoft_defender") +microsoft_defender_query_mappings = MicrosoftDefenderMappings( + platform_dir="microsoft_defender", platform_details=microsoft_defender_query_details +) diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_defender.py index a903f0b3..99fc551d 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_defender.py @@ -18,14 +18,14 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager -from app.translator.platforms.microsoft.const import microsoft_defender_details +from app.translator.platforms.microsoft.const import microsoft_defender_query_details from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions -from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings +from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_query_mappings from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser @parser_manager.register_supported_by_roota class MicrosoftDefenderQueryParser(MicrosoftSentinelQueryParser): - mappings: MicrosoftDefenderMappings = microsoft_defender_mappings - details: PlatformDetails = microsoft_defender_details + mappings: MicrosoftDefenderMappings = microsoft_defender_query_mappings + details: PlatformDetails = microsoft_defender_query_details platform_functions: MicrosoftFunctions = microsoft_defender_functions diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index 507c8c17..24d522e9 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -23,14 +23,14 @@ from app.translator.managers import parser_manager from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_sentinel_functions -from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings +from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_query_mappings from app.translator.platforms.microsoft.tokenizer import MicrosoftSentinelTokenizer @parser_manager.register_supported_by_roota class MicrosoftSentinelQueryParser(PlatformQueryParser): platform_functions: MicrosoftFunctions = microsoft_sentinel_functions - mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings + mappings: MicrosoftSentinelMappings = microsoft_sentinel_query_mappings tokenizer = MicrosoftSentinelTokenizer() details: PlatformDetails = microsoft_sentinel_query_details @@ -43,10 +43,10 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFun def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources, functions = self._parse_query(query=raw_query_container.query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - fields_tokens = self.get_fields_tokens(tokens=tokens) - self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) + query_tokens = self.get_query_tokens(query) + field_tokens = self.get_field_tokens(query_tokens, functions.functions) + source_mappings = self.get_source_mappings(field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = fields_tokens + meta_info.query_fields = field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] - return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py index 9cf400e2..ab60a21f 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py @@ -21,12 +21,14 @@ from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager from app.translator.platforms.microsoft.const import microsoft_sentinel_rule_details +from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_rule_mappings from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser @parser_manager.register class MicrosoftSentinelRuleParser(MicrosoftSentinelQueryParser, JsonRuleMixin): details: PlatformDetails = microsoft_sentinel_rule_details + mappings: MicrosoftSentinelMappings = microsoft_sentinel_rule_mappings def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: rule = self.load_rule(text=text) diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py index 38617b55..69953044 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py @@ -19,9 +19,9 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.microsoft.const import microsoft_defender_details +from app.translator.platforms.microsoft.const import microsoft_defender_query_details from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions -from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings +from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_query_mappings from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, @@ -29,19 +29,17 @@ class MicrosoftDefenderFieldValueRender(MicrosoftSentinelFieldValueRender): - details: PlatformDetails = microsoft_defender_details + details: PlatformDetails = microsoft_defender_query_details @render_manager.register class MicrosoftDefenderQueryRender(MicrosoftSentinelQueryRender): - mappings: MicrosoftDefenderMappings = microsoft_defender_mappings - details: PlatformDetails = microsoft_defender_details + mappings: MicrosoftDefenderMappings = microsoft_defender_query_mappings + details: PlatformDetails = microsoft_defender_query_details platform_functions: MicrosoftFunctions = None or_token = "or" field_value_render = MicrosoftDefenderFieldValueRender(or_token=or_token) - is_strict_mapping = True - def init_platform_functions(self) -> None: self.platform_functions = microsoft_defender_functions self.platform_functions.platform_query_render = self diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py index 621decb1..72521800 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py @@ -22,13 +22,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render_cti import RenderCTI from app.translator.managers import render_cti_manager -from app.translator.platforms.microsoft.const import microsoft_defender_details +from app.translator.platforms.microsoft.const import microsoft_defender_query_details from app.translator.platforms.microsoft.mappings.mdatp_cti import DEFAULT_MICROSOFT_DEFENDER_MAPPING @render_cti_manager.register class MicrosoftDefenderCTI(RenderCTI): - details: PlatformDetails = microsoft_defender_details + details: PlatformDetails = microsoft_defender_query_details field_value_templates_map: ClassVar[dict[str, str]] = { "default": '{key} =~ "{value}"', diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py index 7ef6f1f9..961fe98a 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py @@ -27,7 +27,7 @@ from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.escape_manager import microsoft_escape_manager from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_sentinel_functions -from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings +from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_query_mappings class MicrosoftSentinelFieldValueRender(BaseFieldValueRender): @@ -130,7 +130,7 @@ class MicrosoftSentinelQueryRender(PlatformQueryRender): field_value_render = MicrosoftSentinelFieldValueRender(or_token=or_token) - mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings + mappings: MicrosoftSentinelMappings = microsoft_sentinel_query_mappings comment_symbol = "//" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index b5631ef5..1a64f14b 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -27,6 +27,7 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details +from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_rule_mappings from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, @@ -49,6 +50,7 @@ class MicrosoftSentinelRuleFieldValueRender(MicrosoftSentinelFieldValueRender): @render_manager.register class MicrosoftSentinelRuleRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_sentinel_rule_details + mappings: MicrosoftSentinelMappings = microsoft_sentinel_rule_mappings or_token = "or" field_value_render = MicrosoftSentinelRuleFieldValueRender(or_token=or_token) @@ -75,6 +77,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -92,4 +95,5 @@ def finalize_query( rule["tactics"] = mitre_tactics rule["techniques"] = mitre_techniques json_rule = json.dumps(rule, indent=4, sort_keys=False) + json_rule = self.wrap_with_unmapped_fields(json_rule, unmapped_fields) return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/opensearch/mapping.py b/uncoder-core/app/translator/platforms/opensearch/mapping.py index 57b4190d..ad0222ed 100644 --- a/uncoder-core/app/translator/platforms/opensearch/mapping.py +++ b/uncoder-core/app/translator/platforms/opensearch/mapping.py @@ -1,8 +1,5 @@ from app.translator.platforms.base.lucene.mapping import LuceneMappings +from app.translator.platforms.opensearch.const import opensearch_query_details, opensearch_rule_details - -class OpenSearchMappings(LuceneMappings): - pass - - -opensearch_mappings = OpenSearchMappings(platform_dir="opensearch") +opensearch_query_mappings = LuceneMappings(platform_dir="opensearch", platform_details=opensearch_query_details) +opensearch_rule_mappings = LuceneMappings(platform_dir="opensearch", platform_details=opensearch_rule_details) diff --git a/uncoder-core/app/translator/platforms/opensearch/parsers/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/parsers/opensearch.py index b07e01f1..6a3a4444 100644 --- a/uncoder-core/app/translator/platforms/opensearch/parsers/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/parsers/opensearch.py @@ -18,12 +18,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.parsers.lucene import LuceneQueryParser from app.translator.platforms.opensearch.const import opensearch_query_details -from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings +from app.translator.platforms.opensearch.mapping import opensearch_query_mappings @parser_manager.register_supported_by_roota class OpenSearchQueryParser(LuceneQueryParser): details: PlatformDetails = opensearch_query_details - mappings: OpenSearchMappings = opensearch_mappings + mappings: LuceneMappings = opensearch_query_mappings diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py index 3298c106..a1a3f1a6 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py @@ -24,9 +24,10 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.opensearch.const import opensearch_query_details -from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings +from app.translator.platforms.opensearch.mapping import opensearch_query_mappings class OpenSearchFieldValueRender(LuceneFieldValueRender): @@ -99,7 +100,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @render_manager.register class OpenSearchQueryRender(LuceneQueryRender): details: PlatformDetails = opensearch_query_details - mappings: OpenSearchMappings = opensearch_mappings + mappings: LuceneMappings = opensearch_query_mappings or_token = "OR" field_value_render = OpenSearchFieldValueRender(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py index 09cd5b62..ac5f74fc 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -21,15 +21,16 @@ import json from typing import Optional, Union +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer +from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.opensearch.const import OPENSEARCH_RULE, opensearch_rule_details -from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings +from app.translator.platforms.opensearch.mapping import opensearch_rule_mappings from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValueRender, OpenSearchQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated AWS OpenSearch Rule" @@ -43,7 +44,7 @@ class OpenSearchRuleFieldValueRender(OpenSearchFieldValueRender): @render_manager.register class OpenSearchRuleRender(OpenSearchQueryRender): details: PlatformDetails = opensearch_rule_details - mappings: OpenSearchMappings = opensearch_mappings + mappings: LuceneMappings = opensearch_rule_mappings or_token = "OR" and_token = "AND" @@ -63,6 +64,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -76,11 +78,12 @@ def finalize_query( rule["triggers"][0]["severity"] = _SEVERITIES_MAP[meta_info.severity] rule["triggers"][0]["actions"][0]["message_template"]["source"] = str(source).replace(", ", ",\n") rule_str = json.dumps(rule, indent=4, sort_keys=False) + rule_str = self.wrap_with_unmapped_fields(rule_str, unmapped_fields) return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) - def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue) and token.field: - for field in self.map_field(token.field, source_mapping): + for field in self.mappings.map_field(token.field, source_mapping): self.fields.update({field: f"{{ctx.results.0.hits.hits.0._source.{field}}}"}) return super().apply_token(token, source_mapping) diff --git a/uncoder-core/app/translator/platforms/palo_alto/mapping.py b/uncoder-core/app/translator/platforms/palo_alto/mapping.py index fc6a7797..3dd5e4c9 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/mapping.py +++ b/uncoder-core/app/translator/platforms/palo_alto/mapping.py @@ -7,6 +7,7 @@ LogSourceSignature, SourceMapping, ) +from app.translator.platforms.palo_alto.const import cortex_xql_query_details class CortexXQLLogSourceSignature(LogSourceSignature): @@ -73,4 +74,6 @@ def get_suitable_source_mappings( return suitable_source_mappings -cortex_xql_mappings = CortexXQLMappings(platform_dir="palo_alto_cortex") +cortex_xql_query_mappings = CortexXQLMappings( + platform_dir="palo_alto_cortex", platform_details=cortex_xql_query_details +) diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index 31bdd1e0..6984b412 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -16,17 +16,16 @@ limitations under the License. ----------------------------------------------------------------- """ - from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE from app.translator.core.context_vars import preset_log_source_str_ctx_var from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager @@ -35,7 +34,7 @@ from app.translator.platforms.palo_alto.mapping import ( CortexXQLLogSourceSignature, CortexXQLMappings, - cortex_xql_mappings, + cortex_xql_query_mappings, ) from app.translator.platforms.palo_alto.str_value_manager import cortex_xql_str_value_manager @@ -166,8 +165,7 @@ class CortexXQLFieldFieldRender(BaseFieldFieldRender): @render_manager.register class CortexXQLQueryRender(PlatformQueryRender): details: PlatformDetails = cortex_xql_query_details - mappings: CortexXQLMappings = cortex_xql_mappings - is_strict_mapping = True + mappings: CortexXQLMappings = cortex_xql_query_mappings predefined_fields_map = PREDEFINED_FIELDS_MAP raw_log_field_patterns_map: ClassVar[dict[str, str]] = { "regex": '| alter {field} = regextract(to_json_string(action_evtlog_data_fields)->{field}{{}}, "\\"(.*)\\"")', @@ -207,7 +205,7 @@ def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, fun log_source_str = preset_log_source_str_ctx_var.get() or str(log_source_signature) return f"{functions_prefix}{log_source_str}" - def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue) and token.field: field_name = token.field.source_name if values_map := SOURCE_MAPPING_TO_FIELD_VALUE_MAP.get(source_mapping.source_id, {}).get(field_name): diff --git a/uncoder-core/app/translator/platforms/qradar/mapping.py b/uncoder-core/app/translator/platforms/qradar/mapping.py new file mode 100644 index 00000000..e179e73b --- /dev/null +++ b/uncoder-core/app/translator/platforms/qradar/mapping.py @@ -0,0 +1,4 @@ +from app.translator.platforms.base.aql.mapping import AQLMappings +from app.translator.platforms.qradar.const import qradar_query_details + +qradar_query_mappings = AQLMappings(platform_dir="qradar", platform_details=qradar_query_details) diff --git a/uncoder-core/app/translator/platforms/qradar/parsers/qradar.py b/uncoder-core/app/translator/platforms/qradar/parsers/qradar.py index c74d3f1f..ddb2f0cb 100644 --- a/uncoder-core/app/translator/platforms/qradar/parsers/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/parsers/qradar.py @@ -18,12 +18,15 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import parser_manager +from app.translator.platforms.base.aql.mapping import AQLMappings from app.translator.platforms.base.aql.parsers.aql import AQLQueryParser from app.translator.platforms.qradar.const import qradar_query_details +from app.translator.platforms.qradar.mapping import qradar_query_mappings @parser_manager.register_supported_by_roota class QradarQueryParser(AQLQueryParser): details: PlatformDetails = qradar_query_details + mappings: AQLMappings = qradar_query_mappings wrapped_with_comment_pattern = r"^\s*/\*(?:|\n|.)*\*/" diff --git a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py index cf4a7d51..a6846dad 100644 --- a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py @@ -19,8 +19,10 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager +from app.translator.platforms.base.aql.mapping import AQLMappings from app.translator.platforms.base.aql.renders.aql import AQLFieldValueRender, AQLQueryRender from app.translator.platforms.qradar.const import qradar_query_details +from app.translator.platforms.qradar.mapping import qradar_query_mappings class QradarFieldValueRender(AQLFieldValueRender): @@ -30,4 +32,5 @@ class QradarFieldValueRender(AQLFieldValueRender): @render_manager.register class QradarQueryRender(AQLQueryRender): details: PlatformDetails = qradar_query_details + mappings: AQLMappings = qradar_query_mappings field_value_render = QradarFieldValueRender(or_token=AQLQueryRender.or_token) diff --git a/uncoder-core/app/translator/platforms/sigma/const.py b/uncoder-core/app/translator/platforms/sigma/const.py index b7f88a98..aaedda41 100644 --- a/uncoder-core/app/translator/platforms/sigma/const.py +++ b/uncoder-core/app/translator/platforms/sigma/const.py @@ -1,3 +1,5 @@ +from app.translator.core.models.platform_details import PlatformDetails + SIGMA_RULE_DETAILS = { "name": "Sigma", "platform_id": "sigma", @@ -5,3 +7,5 @@ "group_name": "Sigma", "group_id": "sigma", } + +sigma_rule_details = PlatformDetails(**SIGMA_RULE_DETAILS) diff --git a/uncoder-core/app/translator/platforms/sigma/mapping.py b/uncoder-core/app/translator/platforms/sigma/mapping.py index 1af791ac..769e5c25 100644 --- a/uncoder-core/app/translator/platforms/sigma/mapping.py +++ b/uncoder-core/app/translator/platforms/sigma/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.sigma.const import sigma_rule_details class SigmaLogSourceSignature(LogSourceSignature): @@ -59,4 +60,4 @@ def get_suitable_source_mappings( return suitable_source_mappings or [self._source_mappings[DEFAULT_MAPPING_NAME]] -sigma_mappings = SigmaMappings(platform_dir="sigma") +sigma_rule_mappings = SigmaMappings(platform_dir="sigma", platform_details=sigma_rule_details) diff --git a/uncoder-core/app/translator/platforms/sigma/models/compiler.py b/uncoder-core/app/translator/platforms/sigma/models/compiler.py index 2c0b6472..c6092498 100644 --- a/uncoder-core/app/translator/platforms/sigma/models/compiler.py +++ b/uncoder-core/app/translator/platforms/sigma/models/compiler.py @@ -19,8 +19,9 @@ from typing import Union from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.keyword import Keyword +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.platforms.sigma.models.group import Group from app.translator.platforms.sigma.models.operator import NOT, Operator diff --git a/uncoder-core/app/translator/platforms/sigma/models/modifiers.py b/uncoder-core/app/translator/platforms/sigma/models/modifiers.py index 446eb310..fa98c8ce 100644 --- a/uncoder-core/app/translator/platforms/sigma/models/modifiers.py +++ b/uncoder-core/app/translator/platforms/sigma/models/modifiers.py @@ -1,8 +1,8 @@ from typing import ClassVar, Optional, Union from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType -from app.translator.core.models.field import FieldValue -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.str_value_manager import StrValue from app.translator.platforms.sigma.str_value_manager import sigma_str_value_manager diff --git a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py index 9f2fd7ab..65ebc822 100644 --- a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py @@ -21,23 +21,24 @@ from app.translator.core.exceptions.core import SigmaRuleValidationException from app.translator.core.mixins.rule import YamlRuleMixin -from app.translator.core.models.field import Field, FieldValue +from app.translator.core.models.query_tokens.field import Field +from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import QueryParser from app.translator.core.tokenizer import QueryTokenizer from app.translator.managers import parser_manager -from app.translator.platforms.sigma.const import SIGMA_RULE_DETAILS -from app.translator.platforms.sigma.mapping import SigmaMappings, sigma_mappings +from app.translator.platforms.sigma.const import sigma_rule_details +from app.translator.platforms.sigma.mapping import SigmaMappings, sigma_rule_mappings from app.translator.platforms.sigma.tokenizer import SigmaConditionTokenizer, SigmaTokenizer @parser_manager.register_main class SigmaParser(QueryParser, YamlRuleMixin): - details: PlatformDetails = PlatformDetails(**SIGMA_RULE_DETAILS) + details: PlatformDetails = sigma_rule_details condition_tokenizer = SigmaConditionTokenizer() tokenizer: SigmaTokenizer = SigmaTokenizer() - mappings: SigmaMappings = sigma_mappings + mappings: SigmaMappings = sigma_rule_mappings mandatory_fields = {"title", "description", "logsource", "detection"} wrapped_with_comment_pattern = r"^\s*#.*(?:\n|$)" diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index 856fd4a3..25494e7f 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -24,14 +24,15 @@ from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping -from app.translator.core.models.field import FieldValue, Keyword +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.keyword import Keyword from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.render import QueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager -from app.translator.platforms.sigma.const import SIGMA_RULE_DETAILS -from app.translator.platforms.sigma.mapping import SigmaLogSourceSignature, SigmaMappings, sigma_mappings +from app.translator.platforms.sigma.const import sigma_rule_details +from app.translator.platforms.sigma.mapping import SigmaLogSourceSignature, SigmaMappings, sigma_rule_mappings from app.translator.platforms.sigma.models.compiler import DataStructureCompiler from app.translator.platforms.sigma.models.group import Group from app.translator.platforms.sigma.models.operator import AND, NOT, OR @@ -50,8 +51,8 @@ class SigmaRender(QueryRender): comment_symbol = "#" is_single_line_comment = True - mappings: SigmaMappings = sigma_mappings - details: PlatformDetails = PlatformDetails(**SIGMA_RULE_DETAILS) + mappings: SigmaMappings = sigma_rule_mappings + details: PlatformDetails = sigma_rule_details str_value_manager = sigma_str_value_manager @property @@ -197,15 +198,8 @@ def generate_not(self, data: Any, source_mapping: SourceMapping): not_node["condition"] = f"not {condition}" return not_node - @staticmethod - def map_field(source_mapping: SourceMapping, generic_field_name: str) -> str: - field_name = source_mapping.fields_mapping.get_platform_field_name(generic_field_name) - return field_name or generic_field_name - def generate_field(self, data: FieldValue, source_mapping: SourceMapping): - source_id = source_mapping.source_id - generic_field_name = data.field.get_generic_field_name(source_id) or data.field.source_name - field_name = self.map_field(source_mapping, generic_field_name) + field_name = self.mappings.map_field(data.field, source_mapping)[0] if data.operator.token_type not in ( OperatorType.EQ, OperatorType.LT, diff --git a/uncoder-core/app/translator/platforms/sigma/tokenizer.py b/uncoder-core/app/translator/platforms/sigma/tokenizer.py index 0893588f..faa0970a 100644 --- a/uncoder-core/app/translator/platforms/sigma/tokenizer.py +++ b/uncoder-core/app/translator/platforms/sigma/tokenizer.py @@ -21,8 +21,9 @@ from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType from app.translator.core.exceptions.parser import TokenizerGeneralException -from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.keyword import Keyword +from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.platforms.sigma.models.modifiers import ModifierManager diff --git a/uncoder-core/app/translator/platforms/splunk/mapping.py b/uncoder-core/app/translator/platforms/splunk/mapping.py index 1851b8af..5559a947 100644 --- a/uncoder-core/app/translator/platforms/splunk/mapping.py +++ b/uncoder-core/app/translator/platforms/splunk/mapping.py @@ -1,6 +1,7 @@ from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping +from app.translator.platforms.splunk.const import splunk_alert_details, splunk_query_details class SplunkLogSourceSignature(LogSourceSignature): @@ -69,4 +70,5 @@ def get_suitable_source_mappings( return suitable_source_mappings or [self._source_mappings[DEFAULT_MAPPING_NAME]] -splunk_mappings = SplunkMappings(platform_dir="splunk") +splunk_query_mappings = SplunkMappings(platform_dir="splunk", platform_details=splunk_query_details) +splunk_alert_mappings = SplunkMappings(platform_dir="splunk", platform_details=splunk_alert_details) diff --git a/uncoder-core/app/translator/platforms/splunk/parsers/splunk.py b/uncoder-core/app/translator/platforms/splunk/parsers/splunk.py index e1030b55..2370717a 100644 --- a/uncoder-core/app/translator/platforms/splunk/parsers/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/parsers/splunk.py @@ -21,15 +21,14 @@ from app.translator.platforms.base.spl.parsers.spl import SplQueryParser from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions -from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings +from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_query_mappings @parser_manager.register_supported_by_roota class SplunkQueryParser(SplQueryParser): details: PlatformDetails = splunk_query_details + mappings: SplunkMappings = splunk_query_mappings + platform_functions: SplunkFunctions = splunk_functions log_source_pattern = r"^___source_type___\s*=\s*(?:\"(?P[%a-zA-Z_*:0-9\-/]+)\"|(?P[%a-zA-Z_*:0-9\-/]+))(?:\s+(?:and|or)\s+|\s+)?" # noqa: E501 log_source_key_types = ("index", "source", "sourcetype", "sourcecategory") - - mappings: SplunkMappings = splunk_mappings - platform_functions: SplunkFunctions = splunk_functions diff --git a/uncoder-core/app/translator/platforms/splunk/parsers/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/parsers/splunk_alert.py index 1049ffbf..903478a9 100644 --- a/uncoder-core/app/translator/platforms/splunk/parsers/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/parsers/splunk_alert.py @@ -22,12 +22,14 @@ from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager from app.translator.platforms.splunk.const import splunk_alert_details +from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_alert_mappings from app.translator.platforms.splunk.parsers.splunk import SplunkQueryParser @parser_manager.register class SplunkAlertParser(SplunkQueryParser): details: PlatformDetails = splunk_alert_details + mappings: SplunkMappings = splunk_alert_mappings def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: query = re.search(r"search\s*=\s*(?P.+)", text).group("query") diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py index e14c6bfc..7a50d3d1 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py @@ -22,7 +22,7 @@ from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions -from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings +from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_query_mappings class SplunkFieldValueRender(SplFieldValueRender): @@ -32,12 +32,12 @@ class SplunkFieldValueRender(SplFieldValueRender): @render_manager.register class SplunkQueryRender(SplQueryRender): details: PlatformDetails = splunk_query_details + mappings: SplunkMappings = splunk_query_mappings + platform_functions: SplunkFunctions = None or_token = "OR" field_value_render = SplunkFieldValueRender(or_token=or_token) - mappings: SplunkMappings = splunk_mappings - platform_functions: SplunkFunctions = None def init_platform_functions(self) -> None: self.platform_functions = splunk_functions diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py index 5dc2096a..01c27525 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py @@ -25,6 +25,7 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details +from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_alert_mappings from app.translator.platforms.splunk.renders.splunk import SplunkFieldValueRender, SplunkQueryRender from app.translator.tools.utils import get_rule_description_str @@ -39,6 +40,8 @@ class SplunkAlertFieldValueRender(SplunkFieldValueRender): @render_manager.register class SplunkAlertRender(SplunkQueryRender): details: PlatformDetails = splunk_alert_details + mappings: SplunkMappings = splunk_alert_mappings + or_token = "OR" field_value_render = SplunkAlertFieldValueRender(or_token=or_token) @@ -59,6 +62,7 @@ def finalize_query( meta_info: Optional[MetaInfoContainer] = None, source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 not_supported_functions: Optional[list] = None, + unmapped_fields: Optional[list[str]] = None, *args, # noqa: ARG002 **kwargs, # noqa: ARG002 ) -> str: @@ -74,4 +78,5 @@ def finalize_query( if mitre_techniques: mitre_str = f"action.correlationsearch.annotations = {mitre_techniques})" rule = rule.replace("", mitre_str) + rule = self.wrap_with_unmapped_fields(rule, unmapped_fields) return self.wrap_with_not_supported_functions(rule, not_supported_functions) From 7a6cb30a584ac4d4f17466ba5adab095eff67632 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 23 Jul 2024 10:12:26 +0200 Subject: [PATCH 29/40] upd From bb5d48a1487fcc8e1abf047f37cfe28560c5eada Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Wed, 14 Aug 2024 11:59:11 +0200 Subject: [PATCH 30/40] upd From 5a2fccca6947d6825fbf7547f026a89f5113856b Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Wed, 14 Aug 2024 16:07:37 +0200 Subject: [PATCH 31/40] added unsuported keywords --- .../platforms/elasticsearch/renders/esql.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index b3fcf194..593c4247 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -16,22 +16,26 @@ limitations under the License. ----------------------------------------------------------------- """ -from typing import Union +from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType +from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -from app.translator.platforms.elasticsearch.str_value_manager import ESQLStrValueManager, esql_str_value_manager +from app.translator.platforms.elasticsearch.mapping import LuceneMappings, esql_query_mappings +from app.translator.platforms.elasticsearch.str_value_manager import ( + ESQLQueryStrValueManager, + esql_query_str_value_manager, +) class ESQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = elasticsearch_esql_query_details - str_value_manager: ESQLStrValueManager = esql_str_value_manager + str_value_manager: ESQLQueryStrValueManager = esql_query_str_value_manager @staticmethod def _make_case_insensitive(value: str) -> str: @@ -102,11 +106,14 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: value = pre_processed_value return f'{field} rlike ".*{value}.*"' + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 + raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords") + @render_manager.register class ESQLQueryRender(PlatformQueryRender): details: PlatformDetails = elasticsearch_esql_query_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = esql_query_mappings comment_symbol = "//" or_token = "or" @@ -114,7 +121,7 @@ class ESQLQueryRender(PlatformQueryRender): not_token = "not" field_value_render = ESQLFieldValueRender(or_token=or_token) - def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 + def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) if str(log_source_signature) else "*" return f"FROM {table} metadata _id, _version, _index |" From 917ff0dfe9f70f705b5807f0c6a519961f3466a8 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 20 Aug 2024 16:01:36 +0200 Subject: [PATCH 32/40] update prefixes --- .../platforms/elasticsearch/renders/esql.py | 4 +-- .../elasticsearch/renders/esql_rule.py | 34 +++++++++++-------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 593c4247..786f9f4f 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -54,7 +54,7 @@ def _wrap_str_value(value: str) -> str: def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" - return f"{field} = {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} == {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" def less_modifier(self, field: str, value: Union[int, str]) -> str: return f"{field} < {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" @@ -123,7 +123,7 @@ class ESQLQueryRender(PlatformQueryRender): def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) if str(log_source_signature) else "*" - return f"FROM {table} metadata _id, _version, _index |" + return f"FROM {table} |" @staticmethod def _finalize_search_query(query: str) -> str: diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py index 08edb57a..6eebf0c4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -20,13 +20,13 @@ import json from typing import Optional, Union -from app.translator.core.mapping import SourceMapping +from app.translator.core.mapping import LogSourceSignature, SourceMapping from app.translator.core.mitre import MitreConfig from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.models.query_container import MetaInfoContainer +from app.translator.core.models.query_container import MetaInfoContainer, MitreInfoContainer from app.translator.managers import render_manager from app.translator.platforms.elasticsearch.const import ESQL_RULE, elasticsearch_esql_rule_details -from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings +from app.translator.platforms.elasticsearch.mapping import LuceneMappings, esql_query_mappings from app.translator.platforms.elasticsearch.renders.esql import ESQLFieldValueRender, ESQLQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated ESQL Rule" @@ -39,31 +39,35 @@ class ESQLRuleFieldValueRender(ESQLFieldValueRender): @render_manager.register class ESQLRuleRender(ESQLQueryRender): details: PlatformDetails = elasticsearch_esql_rule_details - mappings: ElasticSearchMappings = elasticsearch_mappings + mappings: LuceneMappings = esql_query_mappings mitre: MitreConfig = MitreConfig() or_token = "or" field_value_render = ESQLRuleFieldValueRender(or_token=or_token) - def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: - if not mitre_attack.get("techniques"): + def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002 + table = str(log_source_signature) if str(log_source_signature) else "*" + return f"FROM {table} metadata _id, _version, _index |" + + def __create_mitre_threat(self, mitre_attack: MitreInfoContainer) -> Union[list, list[dict]]: + if not mitre_attack.techniques: return [] threat = [] - for tactic in mitre_attack["tactics"]: - tactic_render = {"id": tactic["external_id"], "name": tactic["tactic"], "reference": tactic["url"]} + for tactic in mitre_attack.tactics: + tactic_render = {"id": tactic.external_id, "name": tactic.name, "reference": tactic.url} sub_threat = {"tactic": tactic_render, "framework": "MITRE ATT&CK", "technique": []} - for technique in mitre_attack["techniques"]: - technique_id = technique["technique_id"].lower() + for technique in mitre_attack.techniques: + technique_id = technique.technique_id.lower() if "." in technique_id: - technique_id = technique_id[: technique["technique_id"].index(".")] + technique_id = technique_id[: technique.technique_id.index(".")] main_technique = self.mitre.get_technique(technique_id) - if tactic["tactic"] in main_technique["tactic"]: + if main_technique and tactic.name in main_technique.tactic: sub_threat["technique"].append( { - "id": main_technique["technique_id"], - "name": main_technique["technique"], - "reference": main_technique["url"], + "id": main_technique.technique_id, + "name": main_technique.name, + "reference": main_technique.url, } ) if len(sub_threat["technique"]) > 0: From cd4b741afe2acafb5e1030db7c9c0496234d784c Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 20 Aug 2024 16:01:44 +0200 Subject: [PATCH 33/40] added esql mapping --- uncoder-core/app/translator/core/render.py | 35 +- .../elasticsearch_esql/azure_mcas.yml | 301 ++++++++++++++ .../elasticsearch_esql/azure_office365.yml | 387 ++++++++++++++++++ .../platforms/elasticsearch_esql/cyberark.yml | 303 ++++++++++++++ .../platforms/elasticsearch_esql/default.yml | 8 + .../elasticsearch_esql/windows_image_load.yml | 302 ++++++++++++++ .../elasticsearch_esql/windows_powershell.yml | 302 ++++++++++++++ .../elasticsearch_esql/windows_ps_module.yml | 302 ++++++++++++++ .../elasticsearch_esql/windows_ps_script.yml | 302 ++++++++++++++ .../elasticsearch_esql/windows_security.yml | 300 ++++++++++++++ .../elasticsearch_esql/windows_sysmon.yml | 302 ++++++++++++++ .../platforms/elasticsearch/mapping.py | 13 + .../platforms/elasticsearch/renders/esql.py | 46 ++- 13 files changed, 2878 insertions(+), 25 deletions(-) create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_mcas.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_office365.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/cyberark.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/default.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_image_load.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_powershell.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_module.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_script.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_security.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_sysmon.yml diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 6158b679..5df2c903 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -78,12 +78,21 @@ def _get_value_type(field_name: str, value: Union[int, str, StrValue], value_typ def _wrap_str_value(value: str) -> str: return value + @staticmethod + def _wrap_int_value(value: int) -> str: + return f'"{value}"' + @staticmethod def _map_bool_value(value: bool) -> str: return "true" if value else "false" def _pre_process_value( - self, field: str, value: Union[int, str, StrValue], value_type: str = ValueType.value, wrap_str: bool = False + self, + field: str, + value: Union[int, str, StrValue], + value_type: str = ValueType.value, + wrap_str: bool = False, + wrap_int: bool = False, ) -> Union[int, str]: value_type = self._get_value_type(field, value, value_type) if isinstance(value, StrValue): @@ -94,7 +103,8 @@ def _pre_process_value( return self._wrap_str_value(value) if wrap_str else value if isinstance(value, bool): return self._map_bool_value(value) - return value + if isinstance(value, int): + return self._wrap_int_value(value) if wrap_int else value def _pre_process_values_list( self, field: str, values: list[Union[int, str, StrValue]], value_type: str = ValueType.value @@ -208,7 +218,7 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions: return query def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[str]]) -> str: - if fields: + if wrap_query_with_meta_info_ctx_var.get() and fields: return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{', '.join(fields)}") return query @@ -216,7 +226,9 @@ def wrap_with_comment(self, value: str) -> str: return f"{self.comment_symbol} {value}" @abstractmethod - def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: + def generate( + self, raw_query_container: RawQueryContainer, tokenized_query_container: Optional[TokenizedQueryContainer] + ) -> str: raise NotImplementedError("Abstract method") @@ -318,7 +330,7 @@ def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer] meta_info_dict = { "name: ": meta_info.title, "uuid: ": meta_info.id, - "author: ": meta_info.author if meta_info.author else "not defined in query/rule", + "author: ": meta_info.author_str or "not defined in query/rule", "licence: ": meta_info.license, } query_meta_info = "\n".join( @@ -370,7 +382,7 @@ def finalize(self, queries_map: dict[str, str]) -> str: return result - def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapping]: + def _get_source_mappings(self, source_mapping_ids: list[str]) -> Optional[list[SourceMapping]]: source_mappings = [] for source_mapping_id in source_mapping_ids: if source_mapping := self.mappings.get_source_mapping(source_mapping_id): @@ -468,8 +480,9 @@ def generate_from_tokenized_query_container(self, query_container: TokenizedQuer raise errors[0] return self.finalize(queries_map) - def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: - if isinstance(query_container, RawQueryContainer): - return self.generate_from_raw_query_container(query_container) - - return self.generate_from_tokenized_query_container(query_container) + def generate( + self, raw_query_container: RawQueryContainer, tokenized_query_container: Optional[TokenizedQueryContainer] + ) -> str: + if tokenized_query_container: + return self.generate_from_tokenized_query_container(tokenized_query_container) + return self.generate_from_raw_query_container(raw_query_container) diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_mcas.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_mcas.yml new file mode 100644 index 00000000..6cd2c7f8 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_mcas.yml @@ -0,0 +1,301 @@ +platform: ElasticSearch ES|QL +source: azure_mcas +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + Name: o365.audit.Name + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_office365.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_office365.yml new file mode 100644 index 00000000..3a3aabbe --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_office365.yml @@ -0,0 +1,387 @@ +platform: ElasticSearch ES|QL +source: azure_office365 +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + GroupName: o365.audit.GroupName + LogonType: o365.audit.LogonType + Source: o365.audit.Source + Status: o365.audit.Status + Actor.ID: o365.audit.Actor.ID + Actor.Type: o365.audit.Actor.Type + ActorContextId: o365.audit.ActorContextId + ActorIpAddress: o365.audit.ActorIpAddress + ActorUserId: o365.audit.ActorUserId + ActorYammerUserId: o365.audit.ActorYammerUserId + AlertEntityId: o365.audit.AlertEntityId + AlertId: o365.audit.AlertId + AlertLinks: o365.audit.AlertLinks + AlertType: o365.audit.AlertType + AppId: o365.audit.AppId + ApplicationDisplayName: o365.audit.ApplicationDisplayName + ApplicationId: o365.audit.ApplicationId + AzureActiveDirectoryEventType: o365.audit.AzureActiveDirectoryEventType + Category: o365.audit.Category + ClientAppId: o365.audit.ClientAppId + ClientIP: o365.audit.ClientIP + ClientIPAddress: o365.audit.ClientIPAddress + ClientInfoString: o365.audit.ClientInfoString + ClientRequestId: o365.audit.ClientRequestId + Comments: o365.audit.Comments + CorrelationId: o365.audit.CorrelationId + CreationTime: o365.audit.CreationTime + CustomUniqueId: o365.audit.CustomUniqueId + Data: o365.audit.Data + DataType: o365.audit.DataType + EntityType: o365.audit.EntityType + ErrorNumber: o365.audit.ErrorNumber + EventData: o365.audit.EventData + EventSource: o365.audit.EventSource + ExceptionInfo: o365.audit.ExceptionInfo + ExchangeMetaData: o365.audit.ExchangeMetaData + ExtendedProperties: o365.audit.ExtendedProperties + ExternalAccess: o365.audit.ExternalAccess + Id: o365.audit.Id + ImplicitShare: o365.audit.ImplicitShare + IncidentId: o365.audit.IncidentId + InterSystemsId: o365.audit.InterSystemsId + InternalLogonType: o365.audit.InternalLogonType + IntraSystemId: o365.audit.IntraSystemId + Item: o365.audit.Item + ItemName: o365.audit.ItemName + ItemType: o365.audit.ItemType + ListId: o365.audit.ListId + ListItemUniqueId: o365.audit.ListItemUniqueId + LogonError: o365.audit.LogonError + LogonUserSid: o365.audit.LogonUserSid + MailboxGuid: o365.audit.MailboxGuid + MailboxOwnerMasterAccountSid: o365.audit.MailboxOwnerMasterAccountSid + MailboxOwnerSid: o365.audit.MailboxOwnerSid + MailboxOwnerUPN: o365.audit.MailboxOwnerUPN + Members: o365.audit.Members + ModifiedProperties: o365.audit.ModifiedProperties + ObjectId: o365.audit.ObjectId + Operation: o365.audit.Operation + OrganizationId: o365.audit.OrganizationId + OrganizationName: o365.audit.OrganizationName + OriginatingServer: o365.audit.OriginatingServer + Parameters: o365.audit.Parameters + PolicyDetails: o365.audit.PolicyDetails + PolicyId: o365.audit.PolicyId + RecordType: o365.audit.RecordType + ResultStatus: o365.audit.ResultStatus + SensitiveInfoDetectionIsIncluded: o365.audit.SensitiveInfoDetectionIsIncluded + SessionId: o365.audit.SessionId + Severity: o365.audit.Severity + SharePointMetaData: o365.audit.SharePointMetaData + Site: o365.audit.Site + SiteUrl: o365.audit.SiteUrl + SourceFileExtension: o365.audit.SourceFileExtension + SourceFileName: o365.audit.SourceFileName + SourceRelativeUrl: o365.audit.SourceRelativeUrl + SupportTicketId: o365.audit.SupportTicketId + Target.ID: o365.audit.Target.ID + Target.Type: o365.audit.Target.Type + TargetContextId: o365.audit.TargetContextId + TargetUserOrGroupName: o365.audit.TargetUserOrGroupName + TargetUserOrGroupType: o365.audit.TargetUserOrGroupType + TeamGuid: o365.audit.TeamGuid + TeamName: o365.audit.TeamName + UniqueSharingId: o365.audit.UniqueSharingId + UserAgent: o365.audit.UserAgent + UserId: o365.audit.UserId + UserKey: o365.audit.UserKey + UserType: o365.audit.UserType + Version: o365.audit.Version + WebId: o365.audit.WebId + Workload: o365.audit.Workload + YammerNetworkId: o365.audit.YammerNetworkId + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/cyberark.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/cyberark.yml new file mode 100644 index 00000000..590ddd86 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/cyberark.yml @@ -0,0 +1,303 @@ +platform: ElasticSearch ES|QL +source: cyberark +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + Issuer: source.user.name + Hostname: agent.hostname + Action: event.action + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/default.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/default.yml new file mode 100644 index 00000000..e8be2b1e --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/default.yml @@ -0,0 +1,8 @@ +platform: ElasticSearch ES|QL +source: default + + +default_log_source: + index: "logs-*" + + diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_image_load.yml new file mode 100644 index 00000000..b1734d59 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_image_load.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_image_load +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + ImageLoaded: + - dll.path + - file.path + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_powershell.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_powershell.yml new file mode 100644 index 00000000..dacb1357 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_powershell.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_powershell +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + CommandLine: + - powershell.command.value + - process.command_line.text + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_module.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_module.yml new file mode 100644 index 00000000..da8b0ae7 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_module.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_ps_module +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + CommandLine: + - powershell.command.value + - process.command_line.text + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_script.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_script.yml new file mode 100644 index 00000000..e6f7a247 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_script.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_ps_script +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + CommandLine: + - powershell.command.value + - process.command_line.text + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_security.yml new file mode 100644 index 00000000..6c07164e --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_security.yml @@ -0,0 +1,300 @@ +platform: ElasticSearch ES|QL +source: windows_security +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + IpAddress: source.ip winlog.event_data.ClientAddress + ProcessName: winlog.event_data.ProcessName process.executable.text + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_sysmon.yml new file mode 100644 index 00000000..bde8fc38 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_sysmon.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_sysmon +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + ImageLoaded: + - winlog.event_data.ImageLoaded + - file.path + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py index b0489fbf..6c02cdbe 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py @@ -1,12 +1,15 @@ from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import ( elastalert_details, + elasticsearch_esql_query_details, elasticsearch_lucene_query_details, elasticsearch_rule_details, kibana_rule_details, xpack_watcher_details, ) +DEFAULT_MAPPING_NAME = "default" + elasticsearch_lucene_query_mappings = LuceneMappings( platform_dir="elasticsearch", platform_details=elasticsearch_lucene_query_details ) @@ -14,3 +17,13 @@ elastalert_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=elastalert_details) kibana_rule_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=kibana_rule_details) xpack_watcher_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=xpack_watcher_details) + + +class ElasticESQLMappings(LuceneMappings): + is_strict_mapping: bool = True + skip_load_default_mappings = True + + +esql_query_mappings = ElasticESQLMappings( + platform_dir="elasticsearch_esql", platform_details=elasticsearch_esql_query_details +) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 786f9f4f..2cff087c 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -26,7 +26,7 @@ from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details -from app.translator.platforms.elasticsearch.mapping import LuceneMappings, esql_query_mappings +from app.translator.platforms.elasticsearch.mapping import ElasticESQLMappings, esql_query_mappings from app.translator.platforms.elasticsearch.str_value_manager import ( ESQLQueryStrValueManager, esql_query_str_value_manager, @@ -54,52 +54,70 @@ def _wrap_str_value(value: str) -> str: def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" - return f"{field} == {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} == {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def less_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} < {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} < {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} <= {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} <= {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def greater_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} > {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} > {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} >= {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} >= {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" - return f"{field} != {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} != {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" if field.endswith(".text"): return self.regex_modifier(field=field, value=value) - return f'{field} like "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}*"' + return f'{field} like "*{self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=False, wrap_int=True + )}*"' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f"ends_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})" + return f"ends_with({field}, {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )})" def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return ( - f"starts_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})" - ) + return f"starts_with({field}, {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )})" def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" - pre_processed_value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=False) + pre_processed_value = self._pre_process_value( + field, value, value_type=ValueType.regex_value, wrap_str=False, wrap_int=True + ) if isinstance(pre_processed_value, str): value = self._make_case_insensitive(pre_processed_value) else: @@ -113,7 +131,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 @render_manager.register class ESQLQueryRender(PlatformQueryRender): details: PlatformDetails = elasticsearch_esql_query_details - mappings: LuceneMappings = esql_query_mappings + mappings: ElasticESQLMappings = esql_query_mappings comment_symbol = "//" or_token = "or" From 9fe06e7b8c8070990736a7995675878548a71884 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Tue, 20 Aug 2024 16:01:48 +0200 Subject: [PATCH 34/40] update esql syntax --- .../platforms/elasticsearch/renders/esql.py | 45 ++++++++----------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 2cff087c..b6ce54f0 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -54,63 +54,54 @@ def _wrap_str_value(value: str) -> str: def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" - return f"{field} == {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} == {value}" def less_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} < {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} < {value}" def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} <= {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} <= {value}" def greater_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} > {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} > {value}" def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} >= {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} >= {value}" def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" - return f"{field} != {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} != {value}" def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" if field.endswith(".text"): return self.regex_modifier(field=field, value=value) - return f'{field} like "*{self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=False, wrap_int=True - )}*"' + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False, wrap_int=True) + return f'{field} like "*{value}*"' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f"ends_with({field}, {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"ends_with({field}, {value})" def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return f"starts_with({field}, {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"starts_with({field}, {value})" def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): From a622fc273f7baf8b06f603ac669753fbb843d0f2 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:12:42 +0200 Subject: [PATCH 35/40] update prefixes --- .../platforms/elasticsearch/renders/esql.py | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index b6ce54f0..786f9f4f 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -26,7 +26,7 @@ from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details -from app.translator.platforms.elasticsearch.mapping import ElasticESQLMappings, esql_query_mappings +from app.translator.platforms.elasticsearch.mapping import LuceneMappings, esql_query_mappings from app.translator.platforms.elasticsearch.str_value_manager import ( ESQLQueryStrValueManager, esql_query_str_value_manager, @@ -54,61 +54,52 @@ def _wrap_str_value(value: str) -> str: def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) - return f"{field} == {value}" + return f"{field} == {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" def less_modifier(self, field: str, value: Union[int, str]) -> str: - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) - return f"{field} < {value}" + return f"{field} < {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) - return f"{field} <= {value}" + return f"{field} <= {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" def greater_modifier(self, field: str, value: Union[int, str]) -> str: - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) - return f"{field} > {value}" + return f"{field} > {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) - return f"{field} >= {value}" + return f"{field} >= {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) - return f"{field} != {value}" + return f"{field} != {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" if field.endswith(".text"): return self.regex_modifier(field=field, value=value) - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False, wrap_int=True) - return f'{field} like "*{value}*"' + return f'{field} like "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}*"' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) - return f"ends_with({field}, {value})" + return f"ends_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})" def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) - return f"starts_with({field}, {value})" + return ( + f"starts_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})" + ) def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" - pre_processed_value = self._pre_process_value( - field, value, value_type=ValueType.regex_value, wrap_str=False, wrap_int=True - ) + pre_processed_value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=False) if isinstance(pre_processed_value, str): value = self._make_case_insensitive(pre_processed_value) else: @@ -122,7 +113,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 @render_manager.register class ESQLQueryRender(PlatformQueryRender): details: PlatformDetails = elasticsearch_esql_query_details - mappings: ElasticESQLMappings = esql_query_mappings + mappings: LuceneMappings = esql_query_mappings comment_symbol = "//" or_token = "or" From 9e4c00931759e71e146e43ac429593141e3aff37 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:12:49 +0200 Subject: [PATCH 36/40] added esql mapping --- .../platforms/elasticsearch/renders/esql.py | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 786f9f4f..2cff087c 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -26,7 +26,7 @@ from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details -from app.translator.platforms.elasticsearch.mapping import LuceneMappings, esql_query_mappings +from app.translator.platforms.elasticsearch.mapping import ElasticESQLMappings, esql_query_mappings from app.translator.platforms.elasticsearch.str_value_manager import ( ESQLQueryStrValueManager, esql_query_str_value_manager, @@ -54,52 +54,70 @@ def _wrap_str_value(value: str) -> str: def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" - return f"{field} == {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} == {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def less_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} < {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} < {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} <= {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} <= {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def greater_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} > {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} > {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} >= {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} >= {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" - return f"{field} != {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)}" + return f"{field} != {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )}" def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" if field.endswith(".text"): return self.regex_modifier(field=field, value=value) - return f'{field} like "*{self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False)}*"' + return f'{field} like "*{self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=False, wrap_int=True + )}*"' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f"ends_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})" + return f"ends_with({field}, {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )})" def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return ( - f"starts_with({field}, {self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True)})" - ) + return f"starts_with({field}, {self._pre_process_value( + field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True + )})" def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" - pre_processed_value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=False) + pre_processed_value = self._pre_process_value( + field, value, value_type=ValueType.regex_value, wrap_str=False, wrap_int=True + ) if isinstance(pre_processed_value, str): value = self._make_case_insensitive(pre_processed_value) else: @@ -113,7 +131,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 @render_manager.register class ESQLQueryRender(PlatformQueryRender): details: PlatformDetails = elasticsearch_esql_query_details - mappings: LuceneMappings = esql_query_mappings + mappings: ElasticESQLMappings = esql_query_mappings comment_symbol = "//" or_token = "or" From 0b5f7c4c3b60edb7b05eb41482c8ac0d2e903a54 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:12:53 +0200 Subject: [PATCH 37/40] update esql syntax --- .../platforms/elasticsearch/renders/esql.py | 45 ++++++++----------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index 2cff087c..b6ce54f0 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -54,63 +54,54 @@ def _wrap_str_value(value: str) -> str: def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" - return f"{field} == {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} == {value}" def less_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} < {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} < {value}" def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} <= {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} <= {value}" def greater_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} > {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} > {value}" def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f"{field} >= {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} >= {value}" def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" - return f"{field} != {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )}" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} != {value}" def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" if field.endswith(".text"): return self.regex_modifier(field=field, value=value) - return f'{field} like "*{self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=False, wrap_int=True - )}*"' + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False, wrap_int=True) + return f'{field} like "*{value}*"' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" - return f"ends_with({field}, {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"ends_with({field}, {value})" def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if field.endswith(".text"): return self.regex_modifier(field=field, value=value) if isinstance(value, list): return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" - return f"starts_with({field}, {self._pre_process_value( - field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True - )})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"starts_with({field}, {value})" def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): From 31bb83f50644ad1453a914a12f3e2f758ba06265 Mon Sep 17 00:00:00 2001 From: Dmytro <47281757+tarnopolskyi@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:12:59 +0200 Subject: [PATCH 38/40] upd --- uncoder-core/app/translator/core/render.py | 3 ++- .../platforms/elasticsearch/renders/esql.py | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 5df2c903..287eb1d4 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -80,7 +80,7 @@ def _wrap_str_value(value: str) -> str: @staticmethod def _wrap_int_value(value: int) -> str: - return f'"{value}"' + return str(value) @staticmethod def _map_bool_value(value: bool) -> str: @@ -105,6 +105,7 @@ def _pre_process_value( return self._map_bool_value(value) if isinstance(value, int): return self._wrap_int_value(value) if wrap_int else value + return value def _pre_process_values_list( self, field: str, values: list[Union[int, str, StrValue]], value_type: str = ValueType.value diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py index b6ce54f0..9882e4e3 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -51,6 +51,10 @@ def _make_case_insensitive(value: str) -> str: def _wrap_str_value(value: str) -> str: return f'"{value}"' + @staticmethod + def _wrap_int_value(value: int) -> str: + return f'"{value}"' + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" @@ -106,13 +110,9 @@ def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" - pre_processed_value = self._pre_process_value( - field, value, value_type=ValueType.regex_value, wrap_str=False, wrap_int=True - ) - if isinstance(pre_processed_value, str): - value = self._make_case_insensitive(pre_processed_value) - else: - value = pre_processed_value + value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=False, wrap_int=True) + if isinstance(value, str): + value = self._make_case_insensitive(value) return f'{field} rlike ".*{value}.*"' def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 From c10b89d6e4aa1c6e3acc2c8d7368b1f567ca1678 Mon Sep 17 00:00:00 2001 From: "dmytro.tarnopolskyi" Date: Wed, 11 Sep 2024 10:18:21 +0200 Subject: [PATCH 39/40] fix conflicts --- uncoder-core/app/translator/core/mapping.py | 2 +- uncoder-core/app/translator/core/mitre.py | 2 +- .../translator/core/models/query_container.py | 3 - .../core/models/query_tokens/field.py | 6 -- .../core/models/query_tokens/field_field.py | 11 ---- .../core/models/query_tokens/field_value.py | 11 ---- .../models/query_tokens/function_value.py | 10 ---- .../translator/platforms/base/aql/mapping.py | 35 +----------- .../platforms/base/spl/renders/spl.py | 10 ---- .../platforms/base/spl/str_value_manager.py | 55 +++++++++++++++++++ .../translator/platforms/logscale/mapping.py | 16 ------ .../translator/platforms/microsoft/mapping.py | 17 ------ .../parsers/microsoft_sentinel_rule.py | 11 ---- .../platforms/sigma/parsers/sigma.py | 12 ---- .../platforms/sigma/renders/sigma.py | 8 --- 15 files changed, 58 insertions(+), 151 deletions(-) create mode 100644 uncoder-core/app/translator/platforms/base/spl/str_value_manager.py diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 1486acad..886cfdc3 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -116,7 +116,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]: default_mapping = SourceMapping(source_id=DEFAULT_MAPPING_NAME) for mapping_dict in self._loader.load_platform_mappings(self._platform_dir): log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict) - if (source_id := mapping_dict["source"]) == DEFAULT_MAPPING_NAME: + if (source_id := mapping_dict.get("source")) == DEFAULT_MAPPING_NAME: default_mapping.log_source_signature = log_source_signature if self.skip_load_default_mappings: continue diff --git a/uncoder-core/app/translator/core/mitre.py b/uncoder-core/app/translator/core/mitre.py index 681054f6..2e86a3be 100644 --- a/uncoder-core/app/translator/core/mitre.py +++ b/uncoder-core/app/translator/core/mitre.py @@ -189,7 +189,7 @@ def __load_mitre_configs_from_files(self) -> None: technique_id=technique_data["technique_id"], name=technique_data["technique"], url=technique_data["url"], - tactic=technique_data["tactic"], + tactic=technique_data.get("tactic", []), ) self.techniques.insert(technique_id, technique) except JSONDecodeError: diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index 8325be23..7dca8e7f 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -8,8 +8,6 @@ from app.translator.core.mapping import DEFAULT_MAPPING_NAME from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_tokens.field import Field -<<<<<<< HEAD -======= @dataclass @@ -46,7 +44,6 @@ def __init__( self.trigger_threshold = trigger_threshold self.query_frequency = query_frequency self.query_period = query_period ->>>>>>> main class MetaInfoContainer: diff --git a/uncoder-core/app/translator/core/models/query_tokens/field.py b/uncoder-core/app/translator/core/models/query_tokens/field.py index 664f9263..557c9d90 100644 --- a/uncoder-core/app/translator/core/models/query_tokens/field.py +++ b/uncoder-core/app/translator/core/models/query_tokens/field.py @@ -1,7 +1,4 @@ -<<<<<<< HEAD -======= from abc import ABC, abstractmethod ->>>>>>> main from typing import Optional from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping @@ -41,8 +38,6 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma class PredefinedField: def __init__(self, name: str): self.name = name -<<<<<<< HEAD -======= class BaseFieldsGetter(ABC): @@ -50,4 +45,3 @@ class BaseFieldsGetter(ABC): @abstractmethod def fields(self) -> list[Field]: raise NotImplementedError("Abstract method") ->>>>>>> main diff --git a/uncoder-core/app/translator/core/models/query_tokens/field_field.py b/uncoder-core/app/translator/core/models/query_tokens/field_field.py index d5805361..41d285c7 100644 --- a/uncoder-core/app/translator/core/models/query_tokens/field_field.py +++ b/uncoder-core/app/translator/core/models/query_tokens/field_field.py @@ -1,16 +1,8 @@ -<<<<<<< HEAD -from app.translator.core.models.query_tokens.field import Alias, Field -from app.translator.core.models.query_tokens.identifier import Identifier - - -class FieldField: -======= from app.translator.core.models.query_tokens.field import Alias, BaseFieldsGetter, Field from app.translator.core.models.query_tokens.identifier import Identifier class FieldField(BaseFieldsGetter): ->>>>>>> main def __init__( self, source_name_left: str, @@ -24,8 +16,6 @@ def __init__( self.operator = operator self.field_right = Field(source_name=source_name_right) if not is_alias_right else None self.alias_right = Alias(name=source_name_right) if is_alias_right else None -<<<<<<< HEAD -======= @property def fields(self) -> list[Field]: @@ -36,4 +26,3 @@ def fields(self) -> list[Field]: fields.append(self.field_right) return fields ->>>>>>> main diff --git a/uncoder-core/app/translator/core/models/query_tokens/field_value.py b/uncoder-core/app/translator/core/models/query_tokens/field_value.py index 30660db7..79dfb720 100644 --- a/uncoder-core/app/translator/core/models/query_tokens/field_value.py +++ b/uncoder-core/app/translator/core/models/query_tokens/field_value.py @@ -1,21 +1,13 @@ from typing import Union from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS -<<<<<<< HEAD -from app.translator.core.models.query_tokens.field import Alias, Field, PredefinedField -======= from app.translator.core.models.query_tokens.field import Alias, BaseFieldsGetter, Field, PredefinedField ->>>>>>> main from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.models.query_tokens.value import Value from app.translator.core.str_value_manager import StrValue -<<<<<<< HEAD -class FieldValue(Value): -======= class FieldValue(BaseFieldsGetter, Value): ->>>>>>> main def __init__( self, source_name: str, @@ -41,10 +33,7 @@ def __repr__(self): return f"{self.predefined_field.name} {self.operator.token_type} {self.values}" return f"{self.field.source_name} {self.operator.token_type} {self.values}" -<<<<<<< HEAD -======= @property def fields(self) -> list[Field]: return [self.field] if self.field else [] ->>>>>>> main diff --git a/uncoder-core/app/translator/core/models/query_tokens/function_value.py b/uncoder-core/app/translator/core/models/query_tokens/function_value.py index 6bc44177..89bcd63b 100644 --- a/uncoder-core/app/translator/core/models/query_tokens/function_value.py +++ b/uncoder-core/app/translator/core/models/query_tokens/function_value.py @@ -2,28 +2,18 @@ from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS from app.translator.core.models.functions.base import Function -<<<<<<< HEAD -======= from app.translator.core.models.query_tokens.field import BaseFieldsGetter, Field ->>>>>>> main from app.translator.core.models.query_tokens.identifier import Identifier from app.translator.core.models.query_tokens.value import Value from app.translator.core.str_value_manager import StrValue -<<<<<<< HEAD -class FunctionValue(Value): -======= class FunctionValue(BaseFieldsGetter, Value): ->>>>>>> main def __init__(self, function: Function, operator: Identifier, value: Union[int, str, StrValue, list, tuple]): super().__init__(value, cast_to_int=operator.token_type not in STR_SEARCH_OPERATORS) self.function = function self.operator = operator -<<<<<<< HEAD -======= @property def fields(self) -> list[Field]: return self.function.fields ->>>>>>> main diff --git a/uncoder-core/app/translator/platforms/base/aql/mapping.py b/uncoder-core/app/translator/platforms/base/aql/mapping.py index f356bd33..984b85f2 100644 --- a/uncoder-core/app/translator/platforms/base/aql/mapping.py +++ b/uncoder-core/app/translator/platforms/base/aql/mapping.py @@ -48,7 +48,7 @@ class AQLMappings(BasePlatformMappings): def prepare_log_source_signature(self, mapping: dict) -> AQLLogSourceSignature: log_source = mapping.get("log_source", {}) - default_log_source = mapping["default_log_source"] + default_log_source = mapping.get("default_log_source") return AQLLogSourceSignature( device_types=log_source.get("devicetype"), categories=log_source.get("category"), @@ -56,36 +56,3 @@ def prepare_log_source_signature(self, mapping: dict) -> AQLLogSourceSignature: qid_event_categories=log_source.get("qideventcategory"), default_source=default_log_source, ) -<<<<<<< HEAD - - def get_suitable_source_mappings( - self, - field_names: list[str], - devicetype: Optional[list[int]] = None, - category: Optional[list[int]] = None, - qid: Optional[list[int]] = None, - qideventcategory: Optional[list[int]] = None, - ) -> list[SourceMapping]: - suitable_source_mappings = [] - for source_mapping in self._source_mappings.values(): - if source_mapping.source_id == DEFAULT_MAPPING_NAME: - continue - - log_source_signature: AQLLogSourceSignature = source_mapping.log_source_signature - if log_source_signature.is_suitable(devicetype, category, qid, qideventcategory): # noqa: SIM102 - if source_mapping.fields_mapping.is_suitable(field_names): - suitable_source_mappings.append(source_mapping) - - if not suitable_source_mappings: - for source_mapping in self._source_mappings.values(): - if source_mapping.source_id == DEFAULT_MAPPING_NAME: - continue - if source_mapping.fields_mapping.is_suitable(field_names): - suitable_source_mappings.append(source_mapping) - - if not suitable_source_mappings: - suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]] - - return suitable_source_mappings -======= ->>>>>>> main diff --git a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py index a0893cb8..c8dffa70 100644 --- a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py @@ -20,15 +20,6 @@ from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE -<<<<<<< HEAD -from app.translator.core.exceptions.render import UnsupportedRenderMethod -from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender -from app.translator.platforms.base.spl.escape_manager import spl_escape_manager - - -class SplFieldValueRender(BaseFieldValueRender): - escape_manager = spl_escape_manager -======= from app.translator.core.custom_types.values import ValueType from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue @@ -47,7 +38,6 @@ def _pre_process_value( ) -> Union[int, str]: value = super()._pre_process_value(field, value, value_type=value_type, wrap_str=wrap_str) return self._wrap_str_value(str(value)) if not isinstance(value, str) else value ->>>>>>> main def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): diff --git a/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py b/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py new file mode 100644 index 00000000..84ebaab7 --- /dev/null +++ b/uncoder-core/app/translator/platforms/base/spl/str_value_manager.py @@ -0,0 +1,55 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2023 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from typing import ClassVar + +from app.translator.core.str_value_manager import BaseSpecSymbol, StrValue, StrValueManager, UnboundLenWildCard +from app.translator.platforms.base.spl.escape_manager import spl_escape_manager + + +class SplStrValueManager(StrValueManager): + escape_manager = spl_escape_manager + str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = {"*": UnboundLenWildCard} + + def from_str_to_container(self, value: str) -> StrValue: + split = [] + prev_char = None + for char in value: + if char == "\\": + if prev_char == "\\": + split.append("\\") + prev_char = None + continue + elif char in self.str_spec_symbols_map: + if prev_char == "\\": + split.append(char) + else: + split.append(self.str_spec_symbols_map[char]()) + elif char in ('"', "=", "|", "<", ">"): + split.append(char) + else: + if prev_char == "\\": + split.append(prev_char) + split.append(char) + + prev_char = char + + return StrValue(self.escape_manager.remove_escape(value), self._concat(split)) + + +spl_str_value_manager = SplStrValueManager() diff --git a/uncoder-core/app/translator/platforms/logscale/mapping.py b/uncoder-core/app/translator/platforms/logscale/mapping.py index f7e463ad..2ca91e99 100644 --- a/uncoder-core/app/translator/platforms/logscale/mapping.py +++ b/uncoder-core/app/translator/platforms/logscale/mapping.py @@ -1,10 +1,6 @@ from typing import Optional -<<<<<<< HEAD -from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping -======= from app.translator.core.mapping import BasePlatformMappings, LogSourceSignature ->>>>>>> main from app.translator.platforms.logscale.const import logscale_alert_details, logscale_query_details @@ -25,17 +21,5 @@ def prepare_log_source_signature(self, mapping: dict) -> LogScaleLogSourceSignat return LogScaleLogSourceSignature(default_source=default_log_source) -<<<<<<< HEAD - if source_mapping.fields_mapping.is_suitable(field_names): - suitable_source_mappings.append(source_mapping) - - if not suitable_source_mappings: - suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]] - - return suitable_source_mappings - - -======= ->>>>>>> main logscale_query_mappings = LogScaleMappings(platform_dir="logscale", platform_details=logscale_query_details) logscale_alert_mappings = LogScaleMappings(platform_dir="logscale", platform_details=logscale_alert_details) diff --git a/uncoder-core/app/translator/platforms/microsoft/mapping.py b/uncoder-core/app/translator/platforms/microsoft/mapping.py index 39933d3d..2ad307b6 100644 --- a/uncoder-core/app/translator/platforms/microsoft/mapping.py +++ b/uncoder-core/app/translator/platforms/microsoft/mapping.py @@ -1,10 +1,6 @@ from typing import Optional -<<<<<<< HEAD -from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping -======= from app.translator.core.mapping import BasePlatformMappings, LogSourceSignature ->>>>>>> main from app.translator.platforms.microsoft.const import ( microsoft_defender_query_details, microsoft_sentinel_query_details, @@ -31,19 +27,6 @@ def prepare_log_source_signature(self, mapping: dict) -> MicrosoftSentinelLogSou return MicrosoftSentinelLogSourceSignature(tables=tables, default_source=default_log_source) -<<<<<<< HEAD - log_source_signature: MicrosoftSentinelLogSourceSignature = source_mapping.log_source_signature - if log_source_signature.is_suitable(table=table) and source_mapping.fields_mapping.is_suitable(field_names): - suitable_source_mappings.append(source_mapping) - - if not suitable_source_mappings: - suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]] - - return suitable_source_mappings - - -======= ->>>>>>> main microsoft_sentinel_query_mappings = MicrosoftSentinelMappings( platform_dir="microsoft_sentinel", platform_details=microsoft_sentinel_query_details ) diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py index 64de7ba5..4a2cf6bf 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py @@ -16,9 +16,6 @@ ----------------------------------------------------------------- """ -<<<<<<< HEAD -from app.translator.core.mixins.rule import JsonRuleMixin -======= from contextlib import suppress from datetime import timedelta from typing import Optional, Union @@ -27,7 +24,6 @@ from isodate.isoerror import ISO8601Error from app.translator.core.mixins.rule import JsonRuleMixin, YamlRuleMixin ->>>>>>> main from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import ( MetaInfoContainer, @@ -36,14 +32,10 @@ RawQueryContainer, ) from app.translator.managers import parser_manager -<<<<<<< HEAD -from app.translator.platforms.microsoft.const import microsoft_sentinel_rule_details -======= from app.translator.platforms.microsoft.const import ( microsoft_sentinel_rule_details, microsoft_sentinel_yaml_rule_details, ) ->>>>>>> main from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_rule_mappings from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser from app.translator.tools.utils import parse_rule_description_str @@ -53,14 +45,11 @@ class MicrosoftSentinelRuleParser(MicrosoftSentinelQueryParser, JsonRuleMixin): details: PlatformDetails = microsoft_sentinel_rule_details mappings: MicrosoftSentinelMappings = microsoft_sentinel_rule_mappings -<<<<<<< HEAD -======= @staticmethod def _parse_timeframe(raw_timeframe: Optional[str]) -> Optional[timedelta]: with suppress(ISO8601Error): return isodate.parse_duration(raw_timeframe) ->>>>>>> main def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: rule = self.load_rule(text=text) diff --git a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py index 6f42b0dc..4f04335a 100644 --- a/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/parsers/sigma.py @@ -17,13 +17,8 @@ ----------------------------------------------------------------- """ -<<<<<<< HEAD -from typing import Union -======= from datetime import timedelta -from re import I from typing import Optional, Union ->>>>>>> main from app.translator.core.exceptions.core import SigmaRuleValidationException from app.translator.core.mixins.rule import YamlRuleMixin @@ -96,10 +91,7 @@ def _get_meta_info( false_positives=self.__parse_false_positives(rule.get("falsepositives")), source_mapping_ids=source_mapping_ids, parsed_logsources=parsed_logsources, -<<<<<<< HEAD -======= timeframe=self.__parse_timeframe(rule.get('detection', {}).get('timeframe')) ->>>>>>> main ) def __validate_rule(self, rule: dict): @@ -135,10 +127,6 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain source_mapping_ids=[source_mapping.source_id for source_mapping in source_mappings], sigma_fields_tokens=sigma_fields_tokens, parsed_logsources=log_sources, -<<<<<<< HEAD - fields_tokens=field_tokens, -======= fields_tokens=field_tokens ->>>>>>> main ), ) diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index 366b6755..51b1b642 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -314,17 +314,9 @@ def generate_from_tokenized_query_container(self, query_container: TokenizedQuer return rule + rendered_not_supported return rule -<<<<<<< HEAD - def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: - if isinstance(query_container, RawQueryContainer): - return self.generate_from_raw_query_container(query_container) - - return self.generate_from_tokenized_query_container(query_container) -======= def generate(self, raw_query_container: RawQueryContainer, tokenized_query_container: Optional[TokenizedQueryContainer]) -> str: if tokenized_query_container: return self.generate_from_tokenized_query_container(tokenized_query_container) return self.generate_from_raw_query_container(raw_query_container) ->>>>>>> main From 7f69b3a52890875e453407024a8486ed7b8cede7 Mon Sep 17 00:00:00 2001 From: "dmytro.tarnopolskyi" Date: Wed, 11 Sep 2024 10:23:05 +0200 Subject: [PATCH 40/40] fix conflicts --- .../app/translator/mappings/platforms/qradar/default.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml index 5b7ddbec..813772e0 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml @@ -14,11 +14,6 @@ field_mapping: - DstPort - DestinationPort - remoteport -<<<<<<< HEAD - dst-hostname: DstHost - src-hostname: SrcHost -======= ->>>>>>> main src-port: - SourcePort - localport @@ -36,10 +31,7 @@ field_mapping: - destinationIP - destinationaddress - destination -<<<<<<< HEAD -======= - DstHost ->>>>>>> main User: - userName - EventUserName pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy