From ab0ef9ebad96e490d03e8bfc32e0085151578578 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 21 Aug 2024 11:36:51 +0300 Subject: [PATCH 01/13] gis-8502 add ElasticSearchRuleTOMLParser --- .../app/translator/core/mixins/rule.py | 12 ++++++ .../platforms/elasticsearch/__init__.py | 4 +- .../platforms/elasticsearch/const.py | 10 +++++ .../elasticsearch/parsers/detection_rule.py | 39 ++++++++++++++++++- 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/uncoder-core/app/translator/core/mixins/rule.py b/uncoder-core/app/translator/core/mixins/rule.py index 8f6bc080..361c2da1 100644 --- a/uncoder-core/app/translator/core/mixins/rule.py +++ b/uncoder-core/app/translator/core/mixins/rule.py @@ -1,6 +1,7 @@ import json from typing import Union +import toml import xmltodict import yaml @@ -50,3 +51,14 @@ def load_rule(text: Union[str, bytes]) -> dict: return xmltodict.parse(text) except Exception as err: raise InvalidXMLStructure(error=str(err)) from err + + +class TOMLRuleMixin: + mitre_config: MitreConfig = MitreConfig() + + @staticmethod + def load_rule(text: str) -> dict: + try: + return toml.loads(text) + except json.JSONDecodeError as err: + raise InvalidJSONStructure(error=str(err)) from err diff --git a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py index 96017e2e..710b75ec 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py @@ -1,4 +1,6 @@ -from app.translator.platforms.elasticsearch.parsers.detection_rule import ElasticSearchRuleParser # noqa: F401 +from app.translator.platforms.elasticsearch.parsers.detection_rule import ( + ElasticSearchRuleTOMLParser, # noqa: F401 +) from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser # noqa: F401 from app.translator.platforms.elasticsearch.renders.detection_rule import ElasticSearchRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/elasticsearch/const.py b/uncoder-core/app/translator/platforms/elasticsearch/const.py index 08409610..90e63b4f 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/const.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/const.py @@ -7,6 +7,7 @@ _ELASTIC_LUCENE_RULE = "elastic-lucene-rule" _ELASTIC_KIBANA_RULE = "elastic-kibana-rule" _ELASTALERT_LUCENE_RULE = "elastalert-lucene-rule" +_ELASTALERT_LUCENE_RULE_TOML = "elasticsearch-rule-toml" _ELASTIC_WATCHER_RULE = "elastic-watcher-rule" ELASTIC_QUERY_TYPES = { @@ -32,6 +33,14 @@ **PLATFORM_DETAILS, } +ELASTICSEARCH_RULE_TOML_DETAILS = { + "platform_id": _ELASTALERT_LUCENE_RULE_TOML, + "name": "Elastic Rule TOML", + "platform_name": "Detection Rule (Lucene) TOML", + "first_choice": 0, + **PLATFORM_DETAILS, +} + KIBANA_DETAILS = { "platform_id": _ELASTIC_KIBANA_RULE, "name": "Elastic Kibana Saved Search", @@ -58,6 +67,7 @@ elasticsearch_lucene_query_details = PlatformDetails(**ELASTICSEARCH_LUCENE_QUERY_DETAILS) elasticsearch_rule_details = PlatformDetails(**ELASTICSEARCH_RULE_DETAILS) +elasticsearch_rule_toml_details = PlatformDetails(**ELASTICSEARCH_RULE_TOML_DETAILS) elastalert_details = PlatformDetails(**ELASTALERT_DETAILS) kibana_rule_details = PlatformDetails(**KIBANA_DETAILS) xpack_watcher_details = PlatformDetails(**XPACK_WATCHER_DETAILS) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py index 91ff35c6..49f0a8eb 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py @@ -15,12 +15,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ +from datetime import datetime -from app.translator.core.mixins.rule import JsonRuleMixin +from app.translator.core.mixins.rule import JsonRuleMixin, TOMLRuleMixin from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager -from app.translator.platforms.elasticsearch.const import elasticsearch_rule_details +from app.translator.platforms.elasticsearch.const import elasticsearch_rule_details, elasticsearch_rule_toml_details from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser from app.translator.tools.utils import parse_rule_description_str @@ -53,3 +54,37 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: mitre_attack=mitre_attack, ), ) + + +@parser_manager.register +class ElasticSearchRuleTOMLParser(ElasticSearchQueryParser, TOMLRuleMixin): + details: PlatformDetails = elasticsearch_rule_toml_details + + def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: + raw_rule = self.load_rule(text=text) + rule = raw_rule.get("rule") + metadata = raw_rule.get("metadata") + mitre_attack = self.mitre_config.get_mitre_info( + tactics=[threat_data["tactic"]["name"].replace(" ", "_").lower() for threat_data in rule.get("threat", [])], + techniques=[threat_data["technique"][0]["id"].lower() for threat_data in rule.get("threat", [])], + ) + if metadata.get("creation_date"): + date = datetime.strptime(metadata.get("creation_date"), "%Y/%m/%d").strftime("%Y-%m-%d") + else: + date = None + return RawQueryContainer( + query=rule["query"], + language=language, + meta_info=MetaInfoContainer( + id_=rule.get("rule_id"), + title=rule.get("name"), + description=rule.get("description"), + author=rule.get("author"), + date=date, + license_=rule.get("license"), + severity=rule.get("severity"), + references=rule.get("references"), + tags=rule.get("tags"), + mitre_attack=mitre_attack, + ), + ) From 65fed850c0560734533eadcfb0b38fd668cdb736 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 21 Aug 2024 15:59:46 +0300 Subject: [PATCH 02/13] gis-8502 fix --- uncoder-core/app/translator/core/exceptions/core.py | 4 ++++ uncoder-core/app/translator/core/mixins/rule.py | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/uncoder-core/app/translator/core/exceptions/core.py b/uncoder-core/app/translator/core/exceptions/core.py index 8a5256e6..9b480c93 100644 --- a/uncoder-core/app/translator/core/exceptions/core.py +++ b/uncoder-core/app/translator/core/exceptions/core.py @@ -88,5 +88,9 @@ class InvalidJSONStructure(InvalidRuleStructure): rule_type: str = "JSON" +class InvalidTOMLStructure(InvalidRuleStructure): + rule_type: str = "TOML" + + class InvalidXMLStructure(InvalidRuleStructure): rule_type: str = "XML" diff --git a/uncoder-core/app/translator/core/mixins/rule.py b/uncoder-core/app/translator/core/mixins/rule.py index 361c2da1..60439f6e 100644 --- a/uncoder-core/app/translator/core/mixins/rule.py +++ b/uncoder-core/app/translator/core/mixins/rule.py @@ -5,7 +5,12 @@ import xmltodict import yaml -from app.translator.core.exceptions.core import InvalidJSONStructure, InvalidXMLStructure, InvalidYamlStructure +from app.translator.core.exceptions.core import ( + InvalidJSONStructure, + InvalidTOMLStructure, + InvalidXMLStructure, + InvalidYamlStructure, +) from app.translator.core.mitre import MitreConfig, MitreInfoContainer @@ -60,5 +65,5 @@ class TOMLRuleMixin: def load_rule(text: str) -> dict: try: return toml.loads(text) - except json.JSONDecodeError as err: - raise InvalidJSONStructure(error=str(err)) from err + except toml.TomlDecodeError as err: + raise InvalidTOMLStructure(error=str(err)) from err From 6890bc46a431a3791450abf3e776c8955284692f Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:02:38 +0300 Subject: [PATCH 03/13] gis-8502 fix --- .../app/translator/platforms/elasticsearch/__init__.py | 1 + uncoder-core/app/translator/platforms/elasticsearch/const.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py index 710b75ec..e28e5519 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py @@ -1,4 +1,5 @@ from app.translator.platforms.elasticsearch.parsers.detection_rule import ( + ElasticSearchRuleParser, ElasticSearchRuleTOMLParser, # noqa: F401 ) from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/elasticsearch/const.py b/uncoder-core/app/translator/platforms/elasticsearch/const.py index 90e63b4f..61ea1897 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/const.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/const.py @@ -5,9 +5,9 @@ _ELASTIC_LUCENE_QUERY = "elastic-lucene-query" _ELASTIC_LUCENE_RULE = "elastic-lucene-rule" +_ELASTIC_LUCENE_RULE_TOML = "elastic-lucene-rule-toml" _ELASTIC_KIBANA_RULE = "elastic-kibana-rule" _ELASTALERT_LUCENE_RULE = "elastalert-lucene-rule" -_ELASTALERT_LUCENE_RULE_TOML = "elasticsearch-rule-toml" _ELASTIC_WATCHER_RULE = "elastic-watcher-rule" ELASTIC_QUERY_TYPES = { @@ -34,7 +34,7 @@ } ELASTICSEARCH_RULE_TOML_DETAILS = { - "platform_id": _ELASTALERT_LUCENE_RULE_TOML, + "platform_id": _ELASTIC_LUCENE_RULE_TOML, "name": "Elastic Rule TOML", "platform_name": "Detection Rule (Lucene) TOML", "first_choice": 0, From 8f7deb4ef1ae77a58e7ecfd64376b519a093dd5c Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:06:41 +0300 Subject: [PATCH 04/13] gis-8503 fix --- .../platforms/elasticsearch/parsers/detection_rule.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py index 49f0a8eb..691c5ed3 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py @@ -68,10 +68,9 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: tactics=[threat_data["tactic"]["name"].replace(" ", "_").lower() for threat_data in rule.get("threat", [])], techniques=[threat_data["technique"][0]["id"].lower() for threat_data in rule.get("threat", [])], ) + date = None if metadata.get("creation_date"): date = datetime.strptime(metadata.get("creation_date"), "%Y/%m/%d").strftime("%Y-%m-%d") - else: - date = None return RawQueryContainer( query=rule["query"], language=language, From b5c1b6587fead94bb9bb4bd7eb2f1f4dd5b2e337 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:13:01 +0300 Subject: [PATCH 05/13] gis-8502 fix --- .../app/translator/core/models/query_container.py | 12 ++++++++++++ .../elasticsearch/parsers/detection_rule.py | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index 719df330..e00b3389 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -36,6 +36,12 @@ def __init__( self, *, id_: Optional[str] = None, + from_: Optional[str] = None, + index: Optional[str] = None, + language: Optional[str] = None, + risk_score: Optional[str] = None, + type_: Optional[str] = None, + interval: Optional[str] = None, title: Optional[str] = None, description: Optional[str] = None, author: Optional[list[str]] = None, @@ -56,6 +62,12 @@ def __init__( ) -> None: self.id = id_ or str(uuid.uuid4()) self.title = title or "" + self.from_ = from_ or "" + self.index = index or "" + self.language = language or "" + self.risk_score = risk_score or "" + self.type_ = type_ or "" + self.interval = interval or "" self.description = description or "" self.author = [v.strip() for v in author] if author else [] self.date = date or datetime.now().date().strftime("%Y-%m-%d") diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py index 691c5ed3..db835a59 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py @@ -85,5 +85,11 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: references=rule.get("references"), tags=rule.get("tags"), mitre_attack=mitre_attack, + from_=rule.get("from"), + index=rule.get("index"), + language=rule.get("language"), + risk_score=rule.get("risk_score"), + type_=rule.get("type"), + interval=rule.get("interval"), ), ) From 78ba241d2234c96ad8aa43d12485dd1d6ab2fbba Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:21:01 +0300 Subject: [PATCH 06/13] gis-8502 change techniques collection --- .../platforms/elasticsearch/parsers/detection_rule.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py index db835a59..45630688 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py @@ -64,9 +64,13 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: raw_rule = self.load_rule(text=text) rule = raw_rule.get("rule") metadata = raw_rule.get("metadata") + techniques = [] + for threat_data in rule.get("threat", []): + if threat_data.get("technique") and len(threat_data.get("technique")) > 0: + techniques.append(threat_data["technique"][0]["id"].lower()) mitre_attack = self.mitre_config.get_mitre_info( tactics=[threat_data["tactic"]["name"].replace(" ", "_").lower() for threat_data in rule.get("threat", [])], - techniques=[threat_data["technique"][0]["id"].lower() for threat_data in rule.get("threat", [])], + techniques=techniques, ) date = None if metadata.get("creation_date"): From fe78dcf88036a9b1d420dbd278e4532ce2fb5099 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Mon, 26 Aug 2024 15:46:47 +0300 Subject: [PATCH 07/13] gis-8502 fix LuceneEscapeManager From 4a31a3d354e82cd1916520a37c587f4d4845394b Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:38:14 +0300 Subject: [PATCH 08/13] Merge branch 'prod' into gis-8502 --- .../app/translator/core/exceptions/core.py | 4 - .../app/translator/core/exceptions/render.py | 2 +- uncoder-core/app/translator/core/mitre.py | 161 +++++--- .../translator/core/models/query_container.py | 19 + uncoder-core/app/translator/core/render.py | 50 ++- .../elasticsearch_esql/azure_mcas.yml | 301 ++++++++++++++ .../elasticsearch_esql/azure_office365.yml | 387 ++++++++++++++++++ .../platforms/elasticsearch_esql/cyberark.yml | 303 ++++++++++++++ .../platforms/elasticsearch_esql/default.yml | 8 + .../elasticsearch_esql/windows_image_load.yml | 302 ++++++++++++++ .../elasticsearch_esql/windows_powershell.yml | 302 ++++++++++++++ .../elasticsearch_esql/windows_ps_module.yml | 302 ++++++++++++++ .../elasticsearch_esql/windows_ps_script.yml | 302 ++++++++++++++ .../elasticsearch_esql/windows_security.yml | 300 ++++++++++++++ .../elasticsearch_esql/windows_sysmon.yml | 302 ++++++++++++++ .../platforms/base/spl/renders/spl.py | 40 +- .../platforms/base/sql/renders/sql.py | 4 - .../platforms/chronicle/renders/chronicle.py | 4 - .../platforms/elasticsearch/__init__.py | 2 + .../platforms/elasticsearch/const.py | 54 +++ .../platforms/elasticsearch/escape_manager.py | 23 ++ .../platforms/elasticsearch/mapping.py | 13 + .../elasticsearch/renders/detection_rule.py | 6 +- .../platforms/elasticsearch/renders/esql.py | 139 +++++++ .../elasticsearch/renders/esql_rule.py | 115 ++++++ .../elasticsearch/str_value_manager.py | 40 ++ .../forti_siem/renders/forti_siem_rule.py | 16 - .../platforms/microsoft/__init__.py | 4 +- .../translator/platforms/microsoft/const.py | 9 + .../parsers/microsoft_sentinel_rule.py | 95 ++++- .../renders/microsoft_sentinel_rule.py | 33 ++ 31 files changed, 3521 insertions(+), 121 deletions(-) create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_mcas.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_office365.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/cyberark.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/default.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_image_load.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_powershell.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_module.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_script.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_security.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_sysmon.yml create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py diff --git a/uncoder-core/app/translator/core/exceptions/core.py b/uncoder-core/app/translator/core/exceptions/core.py index 9b480c93..e6358cce 100644 --- a/uncoder-core/app/translator/core/exceptions/core.py +++ b/uncoder-core/app/translator/core/exceptions/core.py @@ -1,10 +1,6 @@ from typing import Optional -class NotImplementedException(BaseException): - ... - - class BasePlatformException(BaseException): ... diff --git a/uncoder-core/app/translator/core/exceptions/render.py b/uncoder-core/app/translator/core/exceptions/render.py index 4dd14b35..65117d59 100644 --- a/uncoder-core/app/translator/core/exceptions/render.py +++ b/uncoder-core/app/translator/core/exceptions/render.py @@ -14,5 +14,5 @@ class FunctionRenderException(BaseRenderException): class UnsupportedRenderMethod(BaseRenderException): def __init__(self, platform_name: str, method: str): - message = f"Cannot translate. {platform_name} backend does not support {method}." + message = f'Cannot translate. {platform_name} backend does not support "{method}".' super().__init__(message) diff --git a/uncoder-core/app/translator/core/mitre.py b/uncoder-core/app/translator/core/mitre.py index a0f5a144..681054f6 100644 --- a/uncoder-core/app/translator/core/mitre.py +++ b/uncoder-core/app/translator/core/mitre.py @@ -3,7 +3,7 @@ import ssl import urllib.request from json import JSONDecodeError -from typing import Optional +from typing import Optional, Union from urllib.error import HTTPError from app.translator.core.models.query_container import MitreInfoContainer, MitreTacticContainer, MitreTechniqueContainer @@ -11,13 +11,82 @@ from const import ROOT_PROJECT_PATH +class TrieNode: + def __init__(self): + self.children = {} + self.is_end_of_word = False + self.result = None + + +class Trie: + """ + Trie (prefix tree) data structure for storing and searching Mitre ATT&CK Techniques and Tactics strings. + + This class handles the insertion and searching of strings related to Mitre ATT&CK Techniques and Tactics, even when + the strings have variations in spacing, case, or underscores. By normalizing the text—converting it to lowercase and + removing spaces and underscores—different variations of the same logical string are treated as equivalent. + + It means strings 'CredentialAccess', 'credential Access', and 'credential_access' will be processed identically, + leading to the same result. + """ + + def __init__(self): + self.root = TrieNode() + + def normalize_text(self, text: str) -> str: + return text.replace(" ", "").lower().replace("_", "").lower() + + def insert(self, text: str, result: Union[MitreTacticContainer, MitreTechniqueContainer]) -> None: + node = self.root + normalized_text = self.normalize_text(text) + + for char in normalized_text: + if char not in node.children: + node.children[char] = TrieNode() + node = node.children[char] + + node.is_end_of_word = True + node.result = result + + +class TacticsTrie(Trie): + def __init__(self): + self.root = TrieNode() + + def search(self, text: str) -> Optional[MitreTacticContainer]: + node: TrieNode = self.root + normalized_text = self.normalize_text(text) + + for char in normalized_text: + if char not in node.children: + return + node = node.children[char] + + if node.is_end_of_word: + return node.result + + +class TechniquesTrie(Trie): + def search(self, text: str) -> Optional[MitreTechniqueContainer]: + node: TrieNode = self.root + normalized_text = self.normalize_text(text) + + for char in normalized_text: + if char not in node.children: + return + node = node.children[char] + + if node.is_end_of_word: + return node.result + + class MitreConfig(metaclass=SingletonMeta): config_url: str = "https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json" mitre_source_types: tuple = ("mitre-attack",) def __init__(self, server: bool = False): - self.tactics = {} - self.techniques = {} + self.tactics: TacticsTrie = TacticsTrie() + self.techniques: TechniquesTrie = TechniquesTrie() if not server: self.__load_mitre_configs_from_files() @@ -44,7 +113,6 @@ def update_mitre_config(self) -> None: # noqa: PLR0912 return tactic_map = {} - technique_map = {} # Map the tactics for entry in mitre_json["objects"]: @@ -53,11 +121,12 @@ def update_mitre_config(self) -> None: # noqa: PLR0912 for ref in entry["external_references"]: if ref["source_name"] == "mitre-attack": tactic_map[entry["x_mitre_shortname"]] = entry["name"] - self.tactics[entry["name"].replace(" ", "_").lower()] = { - "external_id": ref["external_id"], - "url": ref["url"], - "tactic": entry["name"], - } + + tactic_data = MitreTacticContainer( + external_id=ref["external_id"], url=ref["url"], name=entry["name"] + ) + self.tactics.insert(entry["name"], tactic_data) + break # Map the techniques @@ -68,19 +137,15 @@ def update_mitre_config(self) -> None: # noqa: PLR0912 continue for ref in entry["external_references"]: if ref["source_name"] in self.mitre_source_types: - technique_map[ref["external_id"]] = entry["name"] sub_tactics = [] - # Get Mitre Tactics (Kill-Chains) for tactic in entry["kill_chain_phases"]: if tactic["kill_chain_name"] in self.mitre_source_types: - # Map the short phase_name to tactic name sub_tactics.append(tactic_map[tactic["phase_name"]]) - self.techniques[ref["external_id"].lower()] = { - "technique_id": ref["external_id"], - "technique": entry["name"], - "url": ref["url"], - "tactic": sub_tactics, - } + + technique_data = MitreTechniqueContainer( + technique_id=ref["external_id"], name=entry["name"], url=ref["url"], tactic=sub_tactics + ) + self.techniques.insert(ref["external_id"], technique_data) break # Map the sub-techniques @@ -92,47 +157,49 @@ def update_mitre_config(self) -> None: # noqa: PLR0912 if ref["source_name"] in self.mitre_source_types: sub_technique_id = ref["external_id"] sub_technique_name = entry["name"] - parent_technique_name = technique_map[sub_technique_id.split(".")[0]] - parent_tactics = self.techniques.get(sub_technique_id.split(".")[0].lower(), {}).get( - "tactic", [] - ) - sub_technique_name = f"{parent_technique_name} : {sub_technique_name}" - self.techniques[ref["external_id"].lower()] = { - "technique_id": ref["external_id"], - "technique": sub_technique_name, - "url": ref["url"], - "tactic": parent_tactics, - } + if parent_technique := self.techniques.search(sub_technique_id.split(".")[0]): + sub_technique_name = f"{parent_technique.name} : {sub_technique_name}" + sub_technique_data = MitreTechniqueContainer( + technique_id=ref["external_id"], + name=sub_technique_name, + url=ref["url"], + tactic=parent_technique.tactic, + ) + self.techniques.insert(sub_technique_id, sub_technique_data) break def __load_mitre_configs_from_files(self) -> None: try: with open(os.path.join(ROOT_PROJECT_PATH, "app/dictionaries/tactics.json")) as file: - self.tactics = json.load(file) + loaded = json.load(file) + + for tactic_name, tactic_data in loaded.items(): + tactic = MitreTacticContainer( + external_id=tactic_data["external_id"], url=tactic_data["url"], name=tactic_data["tactic"] + ) + self.tactics.insert(tactic_name, tactic) except JSONDecodeError: - self.tactics = {} + print("Unable to load MITRE Tactics") try: with open(os.path.join(ROOT_PROJECT_PATH, "app/dictionaries/techniques.json")) as file: - self.techniques = json.load(file) + loaded = json.load(file) + for technique_id, technique_data in loaded.items(): + technique = MitreTechniqueContainer( + technique_id=technique_data["technique_id"], + name=technique_data["technique"], + url=technique_data["url"], + tactic=technique_data["tactic"], + ) + self.techniques.insert(technique_id, technique) except JSONDecodeError: - self.techniques = {} + print("Unable to load MITRE Techniques") def get_tactic(self, tactic: str) -> Optional[MitreTacticContainer]: - tactic = tactic.replace(".", "_") - if tactic_found := self.tactics.get(tactic): - return MitreTacticContainer( - external_id=tactic_found["external_id"], url=tactic_found["url"], name=tactic_found["tactic"] - ) + return self.tactics.search(tactic) def get_technique(self, technique_id: str) -> Optional[MitreTechniqueContainer]: - if technique_found := self.techniques.get(technique_id): - return MitreTechniqueContainer( - technique_id=technique_found["technique_id"], - name=technique_found["technique"], - url=technique_found["url"], - tactic=technique_found["tactic"], - ) + return self.techniques.search(technique_id) def get_mitre_info( self, tactics: Optional[list[str]] = None, techniques: Optional[list[str]] = None @@ -140,10 +207,10 @@ def get_mitre_info( tactics_list = [] techniques_list = [] for tactic in tactics or []: - if tactic_found := self.get_tactic(tactic=tactic.lower()): + if tactic_found := self.tactics.search(tactic): tactics_list.append(tactic_found) for technique in techniques or []: - if technique_found := self.get_technique(technique_id=technique.lower()): + if technique_found := self.techniques.search(technique): techniques_list.append(technique_found) return MitreInfoContainer( tactics=sorted(tactics_list, key=lambda x: x.name), diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index e00b3389..76159a4c 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -31,6 +31,21 @@ class MitreInfoContainer: techniques: list[MitreTechniqueContainer] = field(default_factory=list) +class RawMetaInfoContainer: + def __init__( + self, + *, + trigger_operator: Optional[str] = None, + trigger_threshold: Optional[str] = None, + query_frequency: Optional[str] = None, + query_period: Optional[str] = None, + ) -> None: + self.trigger_operator = trigger_operator + self.trigger_threshold = trigger_threshold + self.query_frequency = query_frequency + self.query_period = query_period + + class MetaInfoContainer: def __init__( self, @@ -58,7 +73,9 @@ def __init__( source_mapping_ids: Optional[list[str]] = None, parsed_logsources: Optional[dict] = None, timeframe: Optional[timedelta] = None, + query_period: Optional[timedelta] = None, mitre_attack: MitreInfoContainer = MitreInfoContainer(), + raw_metainfo_container: Optional[RawMetaInfoContainer] = None, ) -> None: self.id = id_ or str(uuid.uuid4()) self.title = title or "" @@ -84,6 +101,8 @@ def __init__( self._source_mapping_ids = source_mapping_ids or [DEFAULT_MAPPING_NAME] self.parsed_logsources = parsed_logsources or {} self.timeframe = timeframe + self.query_period = query_period + self.raw_metainfo_container = raw_metainfo_container @property def author_str(self) -> str: diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 4c057977..8e9f8373 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -27,8 +27,9 @@ from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager -from app.translator.core.exceptions.core import NotImplementedException, StrictPlatformException +from app.translator.core.exceptions.core import StrictPlatformException from app.translator.core.exceptions.parser import UnsupportedOperatorException +from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping from app.translator.core.models.functions.base import Function, RenderedFunctions @@ -78,12 +79,21 @@ def _get_value_type(field_name: str, value: Union[int, str, StrValue], value_typ def _wrap_str_value(value: str) -> str: return value + @staticmethod + def _wrap_int_value(value: int) -> str: + return str(value) + @staticmethod def _map_bool_value(value: bool) -> str: return "true" if value else "false" def _pre_process_value( - self, field: str, value: Union[int, str, StrValue], value_type: str = ValueType.value, wrap_str: bool = False + self, + field: str, + value: Union[int, str, StrValue], + value_type: str = ValueType.value, + wrap_str: bool = False, + wrap_int: bool = False, ) -> Union[int, str]: value_type = self._get_value_type(field, value, value_type) if isinstance(value, StrValue): @@ -94,6 +104,8 @@ def _pre_process_value( return self._wrap_str_value(value) if wrap_str else value if isinstance(value, bool): return self._map_bool_value(value) + if isinstance(value, int): + return self._wrap_int_value(value) if wrap_int else value return value def _pre_process_values_list( @@ -111,55 +123,55 @@ def _pre_process_values_list( return processed def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.EQ.capitalize()) def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.NOT_EQ.capitalize()) def less_modifier(self, field: str, value: Union[int, str]) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.LT.capitalize()) def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.LTE.capitalize()) def greater_modifier(self, field: str, value: Union[int, str]) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.GT.capitalize()) def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.GTE.capitalize()) def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.CONTAINS.capitalize()) def not_contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.NOT_CONTAINS.capitalize()) def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.ENDSWITH.capitalize()) def not_endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.NOT_ENDSWITH.capitalize()) def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.STARTSWITH.capitalize()) def not_startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.NOT_STARTSWITH.capitalize()) def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.REGEX.capitalize()) def not_regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.NOT_REGEX.capitalize()) def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.KEYWORD.capitalize()) def is_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.IS_NONE.capitalize()) def is_not_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise NotImplementedException + raise UnsupportedRenderMethod(platform_name=self.details.name, method=OperatorType.IS_NOT_NONE.capitalize()) def apply_value(self, value: Union[str, int], value_type: str = ValueType.value) -> Union[str, int]: return self.escape_manager.escape(value, value_type) diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_mcas.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_mcas.yml new file mode 100644 index 00000000..6cd2c7f8 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_mcas.yml @@ -0,0 +1,301 @@ +platform: ElasticSearch ES|QL +source: azure_mcas +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + Name: o365.audit.Name + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_office365.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_office365.yml new file mode 100644 index 00000000..3a3aabbe --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/azure_office365.yml @@ -0,0 +1,387 @@ +platform: ElasticSearch ES|QL +source: azure_office365 +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + GroupName: o365.audit.GroupName + LogonType: o365.audit.LogonType + Source: o365.audit.Source + Status: o365.audit.Status + Actor.ID: o365.audit.Actor.ID + Actor.Type: o365.audit.Actor.Type + ActorContextId: o365.audit.ActorContextId + ActorIpAddress: o365.audit.ActorIpAddress + ActorUserId: o365.audit.ActorUserId + ActorYammerUserId: o365.audit.ActorYammerUserId + AlertEntityId: o365.audit.AlertEntityId + AlertId: o365.audit.AlertId + AlertLinks: o365.audit.AlertLinks + AlertType: o365.audit.AlertType + AppId: o365.audit.AppId + ApplicationDisplayName: o365.audit.ApplicationDisplayName + ApplicationId: o365.audit.ApplicationId + AzureActiveDirectoryEventType: o365.audit.AzureActiveDirectoryEventType + Category: o365.audit.Category + ClientAppId: o365.audit.ClientAppId + ClientIP: o365.audit.ClientIP + ClientIPAddress: o365.audit.ClientIPAddress + ClientInfoString: o365.audit.ClientInfoString + ClientRequestId: o365.audit.ClientRequestId + Comments: o365.audit.Comments + CorrelationId: o365.audit.CorrelationId + CreationTime: o365.audit.CreationTime + CustomUniqueId: o365.audit.CustomUniqueId + Data: o365.audit.Data + DataType: o365.audit.DataType + EntityType: o365.audit.EntityType + ErrorNumber: o365.audit.ErrorNumber + EventData: o365.audit.EventData + EventSource: o365.audit.EventSource + ExceptionInfo: o365.audit.ExceptionInfo + ExchangeMetaData: o365.audit.ExchangeMetaData + ExtendedProperties: o365.audit.ExtendedProperties + ExternalAccess: o365.audit.ExternalAccess + Id: o365.audit.Id + ImplicitShare: o365.audit.ImplicitShare + IncidentId: o365.audit.IncidentId + InterSystemsId: o365.audit.InterSystemsId + InternalLogonType: o365.audit.InternalLogonType + IntraSystemId: o365.audit.IntraSystemId + Item: o365.audit.Item + ItemName: o365.audit.ItemName + ItemType: o365.audit.ItemType + ListId: o365.audit.ListId + ListItemUniqueId: o365.audit.ListItemUniqueId + LogonError: o365.audit.LogonError + LogonUserSid: o365.audit.LogonUserSid + MailboxGuid: o365.audit.MailboxGuid + MailboxOwnerMasterAccountSid: o365.audit.MailboxOwnerMasterAccountSid + MailboxOwnerSid: o365.audit.MailboxOwnerSid + MailboxOwnerUPN: o365.audit.MailboxOwnerUPN + Members: o365.audit.Members + ModifiedProperties: o365.audit.ModifiedProperties + ObjectId: o365.audit.ObjectId + Operation: o365.audit.Operation + OrganizationId: o365.audit.OrganizationId + OrganizationName: o365.audit.OrganizationName + OriginatingServer: o365.audit.OriginatingServer + Parameters: o365.audit.Parameters + PolicyDetails: o365.audit.PolicyDetails + PolicyId: o365.audit.PolicyId + RecordType: o365.audit.RecordType + ResultStatus: o365.audit.ResultStatus + SensitiveInfoDetectionIsIncluded: o365.audit.SensitiveInfoDetectionIsIncluded + SessionId: o365.audit.SessionId + Severity: o365.audit.Severity + SharePointMetaData: o365.audit.SharePointMetaData + Site: o365.audit.Site + SiteUrl: o365.audit.SiteUrl + SourceFileExtension: o365.audit.SourceFileExtension + SourceFileName: o365.audit.SourceFileName + SourceRelativeUrl: o365.audit.SourceRelativeUrl + SupportTicketId: o365.audit.SupportTicketId + Target.ID: o365.audit.Target.ID + Target.Type: o365.audit.Target.Type + TargetContextId: o365.audit.TargetContextId + TargetUserOrGroupName: o365.audit.TargetUserOrGroupName + TargetUserOrGroupType: o365.audit.TargetUserOrGroupType + TeamGuid: o365.audit.TeamGuid + TeamName: o365.audit.TeamName + UniqueSharingId: o365.audit.UniqueSharingId + UserAgent: o365.audit.UserAgent + UserId: o365.audit.UserId + UserKey: o365.audit.UserKey + UserType: o365.audit.UserType + Version: o365.audit.Version + WebId: o365.audit.WebId + Workload: o365.audit.Workload + YammerNetworkId: o365.audit.YammerNetworkId + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/cyberark.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/cyberark.yml new file mode 100644 index 00000000..590ddd86 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/cyberark.yml @@ -0,0 +1,303 @@ +platform: ElasticSearch ES|QL +source: cyberark +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + Issuer: source.user.name + Hostname: agent.hostname + Action: event.action + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/default.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/default.yml new file mode 100644 index 00000000..e8be2b1e --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/default.yml @@ -0,0 +1,8 @@ +platform: ElasticSearch ES|QL +source: default + + +default_log_source: + index: "logs-*" + + diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_image_load.yml new file mode 100644 index 00000000..b1734d59 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_image_load.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_image_load +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + ImageLoaded: + - dll.path + - file.path + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_powershell.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_powershell.yml new file mode 100644 index 00000000..dacb1357 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_powershell.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_powershell +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + CommandLine: + - powershell.command.value + - process.command_line.text + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_module.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_module.yml new file mode 100644 index 00000000..da8b0ae7 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_module.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_ps_module +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + CommandLine: + - powershell.command.value + - process.command_line.text + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_script.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_script.yml new file mode 100644 index 00000000..e6f7a247 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_ps_script.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_ps_script +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + CommandLine: + - powershell.command.value + - process.command_line.text + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_security.yml new file mode 100644 index 00000000..6c07164e --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_security.yml @@ -0,0 +1,300 @@ +platform: ElasticSearch ES|QL +source: windows_security +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + IpAddress: source.ip winlog.event_data.ClientAddress + ProcessName: winlog.event_data.ProcessName process.executable.text + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImageLoaded: dll.path + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_sysmon.yml new file mode 100644 index 00000000..bde8fc38 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch_esql/windows_sysmon.yml @@ -0,0 +1,302 @@ +platform: ElasticSearch ES|QL +source: windows_sysmon +log_source: + index: [logs-*] +default_log_source: + index: logs-* +field_mapping: + ImageLoaded: + - winlog.event_data.ImageLoaded + - file.path + ProviderName: + - winlog.event_data.ProviderName + - winlog.provider_name + dns_query_name: dns.question.name + EventID: winlog.event_id + AccessMask: winlog.event_data.AccessMask + AccountName: winlog.event_data.AccountName + AllowedToDelegateTo: winlog.event_data.AllowedToDelegateTo + AttributeLDAPDisplayName: winlog.event_data.AttributeLDAPDisplayName + AttributeValue: winlog.event_data.AttributeValue + AuditPolicyChanges: winlog.event_data.AuditPolicyChanges + AuthenticationPackageName: winlog.event_data.AuthenticationPackageName + CallingProcessName: winlog.event_data.CallingProcessName + CallTrace: winlog.event_data.CallTrace + Channel: winlog.channel + CommandLine: process.command_line.text + Command_Line: process.command_line.text + Commandline: process.command_line.text + commandline: process.command_line.text + ScriptBlockText: powershell.file.script_block_text + Payload: + - powershell.command.invocation_details + - winlog.event_data.Payload + ComputerName: winlog.ComputerName + CurrentDirectory: process.working_directory.text + Description: winlog.event_data.Description + DestinationHostname: + - destination.domain + - dns.question.name + - dns.question.subdomain + DestinationIp: destination.address + dst_ip: destination.address + DestinationPort: destination.port + dst_port: destination.port + DestinationPortName: network.protocol + Details: winlog.event_data.Details + EngineVersion: winlog.event_data.EngineVersion + EventType: winlog.event_data.EventType + FailureCode: winlog.event_data.FailureCode + FileName: file.path.text + GrantedAccess: winlog.event_data.GrantedAccess + GroupName: + - winlog.event_data.GroupName + - group.name + GroupSid: + - group.id + - winlog.event_data.GroupSid + Hashes: winlog.event_data.Hashes + file_hash: winlog.event_data.Hashes + HiveName: winlog.event_data.HiveName + HostVersion: winlog.event_data.HostVersion + Image: process.executable.text + ImagePath: winlog.event_data.ImagePath + Imphash: winlog.event_data.Imphash + IpAddress: source.address + ClientAddress: + - winlog.event_data.ClientAddress + - source.ip + IpPort: source.port + KeyLength: winlog.event_data.KeyLength + LogonProcessName: winlog.event_data.LogonProcessName + LogonType: winlog.event_data.LogonType + MemberName: winlog.event_data.MemberName + MemberSid: winlog.event_data.MemberSid + NewProcessName: winlog.event_data.NewProcessName + ObjectClass: winlog.event_data.ObjectClass + ObjectName: winlog.event_data.ObjectName + ObjectType: winlog.event_data.ObjectType + ObjectValueName: winlog.event_data.ObjectValueName + ParentCommandLine: process.parent.command_line.text + ParentProcessName: process.parent.name.text + ParentImage: process.parent.executable.text + Path: winlog.event_data.Path + PipeName: file.name + ProcessCommandLine: winlog.event_data.ProcessCommandLine + ProcessName: process.executable.text + Properties: winlog.event_data.Properties + RuleName: winlog.event_data.RuleName + RegistryValue: winlog.event_data.RegistryValue + SecurityID: winlog.event_data.SecurityID + ServiceFileName: winlog.event_data.ServiceFileName + ServiceName: winlog.event_data.ServiceName + ShareName: winlog.event_data.ShareName + Signature: winlog.event_data.Signature + Signed: winlog.event_data.Signed + Source: winlog.event_data.Source + SourceHostname: source.domain + SourceImage: process.executable.text + SourceIp: source.address + src_ip: source.address + SourcePort: source.port + src_port: source.port + StartModule: winlog.event_data.StartModule + Status: winlog.event_data.Status + SubStatus: winlog.event_data.SubStatus + SubjectDomainName: winlog.event_data.SubjectDomainName + SubjectUserName: winlog.event_data.SubjectUserName + SubjectUserSid: winlog.event_data.SubjectUserSid + TargetDomainName: winlog.event_data.TargetDomainName + TargetFilename: file.path.text + TargetImage: winlog.event_data.TargetImage + TargetObject: winlog.event_data.TargetObject + TargetSid: winlog.event_data.TargetSid + TargetUserName: winlog.event_data.TargetUserName + TargetUserSid: winlog.event_data.TargetUserSid + QueryName: dns.question.name + TicketEncryptionType: winlog.event_data.TicketEncryptionType + TicketOptions: winlog.event_data.TicketOptions + User: user.name + WorkstationName: source.domain + TransmittedServices: winlog.event_data.TransmittedServices + AuthenticationAlgorithm: winlog.event_data.AuthenticationAlgorithm + BSSID: winlog.event_data.BSSID + BSSType: winlog.event_data.BSSType + CipherAlgorithm: winlog.event_data.CipherAlgorithm + ConnectionId: winlog.event_data.ConnectionId + ConnectionMode: winlog.event_data.ConnectionMode + InterfaceDescription: winlog.event_data.InterfaceDescription + InterfaceGuid: winlog.event_data.InterfaceGuid + OnexEnabled: winlog.event_data.OnexEnabled + PHYType: winlog.event_data.PHYType + ProfileName: winlog.event_data.ProfileName + SSID: winlog.event_data.SSID + QueryResults: dns.answers + OriginalFileName: winlog.event_data.OriginalFileName + Domain: winlog.event_data.Domain + ServiceType: winlog.event_data.ServiceType + SourceName: winlog.event_data.SourceName + StartType: winlog.event_data.StartType + UserID: winlog.event_data.UserID + Initiated: winlog.event_data.Initiated + NewUACList: winlog.event_data.NewUACList + UserAccountControl: winlog.event_data.UserAccountControl + NewUacValue: winlog.event_data.NewUacValue + OldUacValue: winlog.event_data.OldUacValue + AccountExpires: winlog.event_data.AccountExpires + DisplayName: winlog.event_data.DisplayName + DnsHostName: winlog.event_data.DnsHostName + HomeDirectory: winlog.event_data.HomeDirectory + HomePath: winlog.event_data.HomePath + LogonHours: winlog.event_data.LogonHours + PasswordLastSet: winlog.event_data.PasswordLastSet + PrimaryGroupId: winlog.event_data.PrimaryGroupId + PrivilegeList: winlog.event_data.PrivilegeList + ProfilePath: winlog.event_data.ProfilePath + SamAccountName: winlog.event_data.SamAccountName + ScriptPath: winlog.event_data.ScriptPath + ServicePrincipalNames: winlog.event_data.ServicePrincipalNames + SidHistory: winlog.event_data.SidHistory + UserParameters: winlog.event_data.UserParameters + UserPrincipalName: winlog.event_data.UserPrincipalName + UserWorkstations: winlog.event_data.UserWorkstations + RelativeTargetName: winlog.event_data.RelativeTargetName + NotificationPackageName: winlog.event_data.NotificationPackageName + SecurityPackageName: winlog.event_data.SecurityPackageName + HostApplication: process.command_line.text + TaskName: winlog.event_data.TaskName + TaskContent: winlog.event_data.TaskContent + ObjectServer: winlog.event_data.ObjectServer + NewSd: winlog.event_data.NewSd + OldSd: winlog.event_data.OldSd + TestSigning: winlog.event_data.TestSigning + AdvancedOptions: winlog.event_data.AdvancedOptions + ConfigAccessPolicy: winlog.event_data.ConfigAccessPolicy + DisableIntegrityChecks: winlog.event_data.DisableIntegrityChecks + FlightSigning: winlog.event_data.FlightSigning + HypervisorDebug: winlog.event_data.HypervisorDebug + HypervisorLaunchType: winlog.event_data.HypervisorLaunchType + HypervisorLoadOptions: winlog.event_data.HypervisorLoadOptions + KernelDebug: winlog.event_data.KernelDebug + LoadOptions: winlog.event_data.LoadOptions + RemoteEventLogging: winlog.event_data.RemoteEventLogging + ExceptionCode: winlog.event_data.ExceptionCode + CertSerialNumber: winlog.event_data.CertSerialNumber + CertThumbprint: winlog.event_data.CertThumbprint + CertIssuerName: winlog.event_data.CertIssuerName + TicketOptionsDescription: winlog.event_data.TicketOptionsDescription + keywords: winlog.keywords + StartAddress: winlog.event_data.StartAddress + ServiceSid: winlog.event_data.ServiceSid + TargetInfo: winlog.event_data.TargetInfo + ClientProcessId: winlog.event_data.ClientProcessId + ParentProcessId: winlog.event_data.ParentProcessId + AccessList: winlog.event_data.AccessList + GroupMembership: winlog.event_data.GroupMembership + FilterName: winlog.event_data.FilterName + ChangeType: winlog.event_data.ChangeType + LayerName: winlog.event_data.LayerName + ProcessId: winlog.event_data.ProcessId + ProcessID: winlog.event_data.ProcessID + SubjectLogonId: winlog.event_data.SubjectLogonId + ElevatedToken: winlog.event_data.ElevatedToken + PublishURLs: winlog.event_data.PublishURLs + VMUserAuthenticationEvent: horizon.user_authentication_event + VMUserAuthenticationUser: horizon.user_authentication_user + VMUserAuthenticationSourceIp: horizon.user_authentication_source_ip + VMUserAuthenticationTimeStamp: horizon.user_authentication_time_stamp + VMDesktopSessionStartEvent: horizon.desktop_session_start_event + VMDesktopSessionStartUser: horizon.desktop_session_start_user + VMDesktopSessionStartDesktopID: horizon.desktop_session_start_desktop_id + VMDesktopSessionStartTimeStamp: horizon.desktop_session_time_stamp + VMApplicationLaunchEvent: horizon.application_launch_event + VMApplicationLaunchUser: horizon.application_launch_user + VMApplicationLaunchAppId: horizon.application_launch_app_id + VMApplicationLaunchAppName: horizon.application_launch_app_name + VMApplicationLaunchTimeStamp: horizon.application_launch_time_stamp + VMConnectionServerStatusEvent: horizon.connection_server_status_event + VMConnectionServerStatusServer: horizon.connection_server_status_server + VMConnectionServerStatus: horizon.connection_Server_status + VMConnectionServerStatusTimeStamp: horizon.connection_server_status_time_stamp + VMVirtualDesktopPoolManagmentEvent: horizon.virtual_desktop_pool_managment_event + VMVirtualDesktopPoolManagmentPoolId: horizon.virtual_desktop_pool_managment_pool_id + VMVirtualDesktopPoolManagmentPoolName: horizon.virtual_desktop_pool_managment_pool_name + VMVirtualDesktopPoolManagmentTimeStamp: horizon.virtual_desktop_pool_managment_time_stamp + VMLoadBalancingEvent: horizon.load_balancing_event + VMLoadBalancingStatus: horizon.load_balancing_status + VMLoadBalancingAlgorithm: horizon.load_balancing_algorithm + VMLoadBalancingTimeStamp: horizon.load_balancing_time_stamp + VMBlastProtocolEvent: horizon.blast_protocol_event + VMBlastProtocolUser: horizon.blast_protocol_user + VMBlastProtocolProtocolVersion: horizon.blast_protocol_protocol_version + VMBlastProtocolTimeStamp: horizon.blast_protocol_time_stamp + VMSecurityEventName: horizon.security_event_name + VMSecurityEventUser: horizon.security_event_user + VMSecurityEventAlertType: horizon.security_event_alert_type + VMSecurityEventSourceIp: horizon.security_event_source_ip + VMSecurityEventTimeStamp: horizon.security_event_time_stamp + VMLicensingInformationEvent: horizon.licensing_information_event + VMLicensingInformationLicenseType: horizon.licensing_information_license_type + VMLicensingInformationExpiryDate: horizon.licensing_information_expiry_date + VMLicensingInformationTimeStamp: horizon.licensing_information_time_stamp + VMConnectionBrokeringEvent: horizon.connection_brokering_event + VMConnectionBrokeringUser: horizon.connection_brokering_user + VMConnectionBrokeringDesktopId: horizon.connection_brokering_desktop_id + VMConnectionBrokeringStatus: horizon.connection_brokering_status + VMConnectionBrokeringTimeStamp: horizon.connection_brokering_time_stamp + DatastoreName: vsphere.datastore_name + FilesystemType: vsphere.datastore_fstype + DatastoreBytes: vsphere.datastore_capacity_free_bytes + DatastoreBytesUsed: vsphere.datastore_capacity_used_pct + HostName: vsphere.host_name + UsedCPUmhz: vsphere.host_cpu_free_mhz + UsedMemoryBites: vsphere.host_memory_total_bytes + FreeMemoryBites: vsphere.host_memory_free_bytes + VMHostID: vsphere.virtualmachine_host_id + VMHostName: + - vsphere.virtualmachine_host_hostname + - esxi.vmhost_name + VMName: + - vsphere.virtualmachine_name + - esxi.vmname + VMOperatingSystem: vsphere.virtualmachine_os + VMUsedCPU: vsphere.virtualmachine_cpu_used_mhz + VMTotalCPU: vsphere.virtualmachine_cpu_free_mhz + VMMemoryGuestUsed: vsphere.virtualmachine_memory_used_guest_bytes + VMUMemoryHostUsed: vsphere.virtualmachine_memory_used_host_bytes + VMTotalMemoryGuestBytes: vsphere.virtualmachine_memory_total_guest_bytes + VMMemoryGuestFree: vsphere.virtualmachine_memory_free_guest_bytes + VMCustomFields: vsphere.virtualmachine_custom_fields + VMNetworkNames: vsphere.virtualmachine_network_names + VMLogicalSwitchingEvent: nsxv.vmlogical_switching_event + VMLogicalSwitchingEventID: nsxv.vmlogical_switching_event_id + VMLogicalSwitchingName: nsxv.vmlogical_switching_name + VMDistributedFirewallEvent: nsxv.distributed_firewall_event_type + VMDistributedFirewallRuleID: nsxv.distributed_firewall_rule_id + VMDistributedFirewallAction: nsxv.distributed_firewall_action + VMDistributedFirewallSourceIp: nsxv.distributed_firewall_source_ip + VMDistributedFirewallDestinationIp: nsxv.distributed_firewall_destination_ip + VMSecurityGroupEventType: nsxv.security_group_event_type + VMSecurityGroupId: nsxv.security_group_id + VMSecurityGroupName: nsxv.security_group_name + VMEdgeServicesGatewayEventType: nsxv.security_edge_services_gateway_event_type + VMEdgeServicesGatewayESGID: nsxv.security_edge_services_gateway_esgid + VMEdgeServicesGatewayStatus: nsxv.security_edge_services_gateway_status + VMLoadBalancingEventType: nsxv.load_balancing_event_type + VMLoadBalancingId: nsxv.load_balancing_id + VMLoadBalancingVirtualServer: nsxv.load_balancing_virtual_server + VMNSXManagerEventType: nsxv.nsx_manager_event_type + VMNSXManagerEventDescription: nsxv.nsx_manager_event_description + VMEdgeFirewallEventType: nsxv.edge_firewall_event_type + VMEdgeFirewallSourceIP: nsxv.edge_firewall_source_ip + VMEdgeFirewallDestinationIP: nsxv.edge_firewall_destination_ip + VMEdgeFirewallRuleID: nsxv.edge_firewall_rule_id + VMSSLVPNEventType: nsxv.ssl_vpn_event_type + VMSSLVPNUserName: nsxv.ssl_vpn_user_name + VMSSLVPNSourceIp: nsxv.ssl_vpn_source_ip + VMNSXControllerEventType: nsxv.nsx_controller_event_type + VMNSXControllerID: nsxv.nsx_controller_id + VMNSXControllerStatus: nsxv.nsx_controller_status + VMLogicalRoutingEventType: nsxv.logical_routing_event_type + VMLogicalRoutingRouterID: nsxv.logical_routing_router_id + VMLogicalRoutingRouterName: nsxv.logical_routing_router_name diff --git a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py index 74adf32b..c8dffa70 100644 --- a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py @@ -20,58 +20,66 @@ from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.exceptions.render import UnsupportedRenderMethod +from app.translator.core.custom_types.values import ValueType from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender -from app.translator.platforms.base.spl.escape_manager import spl_escape_manager +from app.translator.core.str_value_manager import StrValue +from app.translator.platforms.base.spl.str_value_manager import spl_str_value_manager class SplFieldValueRender(BaseFieldValueRender): - escape_manager = spl_escape_manager + str_value_manager = spl_str_value_manager + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f'"{value}"' + + def _pre_process_value( + self, field: str, value: Union[int, str, StrValue], value_type: str = ValueType.value, wrap_str: bool = False + ) -> Union[int, str]: + value = super()._pre_process_value(field, value, value_type=value_type, wrap_str=wrap_str) + return self._wrap_str_value(str(value)) if not isinstance(value, str) else value def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" - return f'{field}="{self.apply_value(value)}"' + return f"{field}={self._pre_process_value(field, value, wrap_str=True)}" def less_modifier(self, field: str, value: Union[int, str]) -> str: - return f'{field}<"{self.apply_value(value)}"' + return f"{field}<{self._pre_process_value(field, value, wrap_str=True)}" def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f'{field}<="{self.apply_value(value)}"' + return f"{field}<={self._pre_process_value(field, value, wrap_str=True)}" def greater_modifier(self, field: str, value: Union[int, str]) -> str: - return f'{field}>"{self.apply_value(value)}"' + return f"{field}>{self._pre_process_value(field, value, wrap_str=True)}" def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - return f'{field}>="{self.apply_value(value)}"' + return f"{field}>={self._pre_process_value(field, value, wrap_str=True)}" def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" - return f'{field}!="{self.apply_value(value)}"' + return f"{field}!={self._pre_process_value(field, value, wrap_str=True)}" def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.contains_modifier(field=field, value=v) for v in value])})" - return f'{field}="*{self.apply_value(value)}*"' + return f'{field}="*{self._pre_process_value(field, value)}*"' def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.endswith_modifier(field=field, value=v) for v in value])})" - return f'{field}="*{self.apply_value(value)}"' + return f'{field}="*{self._pre_process_value(field, value)}"' def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.startswith_modifier(field=field, value=v) for v in value])})" - return f'{field}="{self.apply_value(value)}*"' + return f'{field}="{self._pre_process_value(field, value)}*"' def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join(self.keywords(field=field, value=v) for v in value)})" - return f'"{self.apply_value(value)}"' - - def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise UnsupportedRenderMethod(platform_name=self.details.name, method="Regex Expression") + return f"{self._pre_process_value(field, value, wrap_str=True)}" class SplQueryRender(PlatformQueryRender): diff --git a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py index d69f1590..9426c0cc 100644 --- a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py @@ -20,7 +20,6 @@ from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import LogSourceSignature from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender @@ -68,9 +67,6 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" return f"{field} ILIKE '{value}' ESCAPE '\\'" - def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords") - class SqlQueryRender(PlatformQueryRender): or_token = "OR" diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py index 7642929f..50fd5cbf 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py @@ -21,7 +21,6 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager @@ -94,9 +93,6 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" return f"{field} = /{self.apply_asterisk_value(value)}/ nocase" - def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords") - @render_manager.register class ChronicleQueryRender(PlatformQueryRender): diff --git a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py index e28e5519..e38205d8 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py @@ -7,5 +7,7 @@ from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elasticsearch_cti import ElasticsearchCTI # noqa: F401 +from app.translator.platforms.elasticsearch.renders.esql import ESQLQueryRender # noqa: F401 +from app.translator.platforms.elasticsearch.renders.esql_rule import ESQLRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.kibana import KibanaRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.xpack_watcher import XPackWatcherRuleRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/elasticsearch/const.py b/uncoder-core/app/translator/platforms/elasticsearch/const.py index 61ea1897..b48c4f0b 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/const.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/const.py @@ -9,6 +9,8 @@ _ELASTIC_KIBANA_RULE = "elastic-kibana-rule" _ELASTALERT_LUCENE_RULE = "elastalert-lucene-rule" _ELASTIC_WATCHER_RULE = "elastic-watcher-rule" +_ELASTIC_ESQL_QUERY = "elastic-esql-query" +_ELASTIC_ESQL_RULE = "elastic-esql-rule" ELASTIC_QUERY_TYPES = { _ELASTIC_LUCENE_QUERY, @@ -16,6 +18,8 @@ _ELASTIC_KIBANA_RULE, _ELASTALERT_LUCENE_RULE, _ELASTIC_WATCHER_RULE, + _ELASTIC_ESQL_QUERY, + _ELASTIC_ESQL_RULE, } ELASTICSEARCH_LUCENE_QUERY_DETAILS = { @@ -25,6 +29,20 @@ **PLATFORM_DETAILS, } +ELASTICSEARCH_ESQL_QUERY_DETAILS = { + "platform_id": _ELASTIC_ESQL_QUERY, + "name": "Elasticsearch ES|QL Query", + "platform_name": "Query (ES|QL)", + **PLATFORM_DETAILS, +} + +ELASTICSEARCH_ESQL_RULE_DETAILS = { + "platform_id": _ELASTIC_ESQL_RULE, + "name": "Elasticsearch ES|QL Rule", + "platform_name": "Rule (ES|QL)", + **PLATFORM_DETAILS, +} + ELASTICSEARCH_RULE_DETAILS = { "platform_id": _ELASTIC_LUCENE_RULE, "name": "Elastic Rule", @@ -66,6 +84,8 @@ } elasticsearch_lucene_query_details = PlatformDetails(**ELASTICSEARCH_LUCENE_QUERY_DETAILS) +elasticsearch_esql_query_details = PlatformDetails(**ELASTICSEARCH_ESQL_QUERY_DETAILS) +elasticsearch_esql_rule_details = PlatformDetails(**ELASTICSEARCH_ESQL_RULE_DETAILS) elasticsearch_rule_details = PlatformDetails(**ELASTICSEARCH_RULE_DETAILS) elasticsearch_rule_toml_details = PlatformDetails(**ELASTICSEARCH_RULE_TOML_DETAILS) elastalert_details = PlatformDetails(**ELASTALERT_DETAILS) @@ -177,3 +197,37 @@ } }, } + +ESQL_RULE = { + "name": "", + "tags": [], + "interval": "5m", + "enabled": True, + "revision": 0, + "description": "", + "risk_score": 21, + "severity": "low", + "license": "", + "output_index": "", + "meta": {"from": "1m"}, + "author": [], + "false_positives": [], + "from": "now-360s", + "rule_id": "", + "max_signals": 100, + "risk_score_mapping": [], + "severity_mapping": [], + "threat": [], + "to": "now", + "references": [], + "version": 1, + "exceptions_list": [], + "immutable": False, + "related_integrations": [], + "required_fields": [], + "setup": "", + "type": "esql", + "language": "esql", + "query": "", + "actions": [], +} diff --git a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py new file mode 100644 index 00000000..2109ed2e --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py @@ -0,0 +1,23 @@ +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.escape_manager import EscapeManager +from app.translator.core.models.escape_details import EscapeDetails + + +class ESQLQueryEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [ + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\\\\\"), + EscapeDetails(pattern=r'"', escape_symbols=r"\""), + EscapeDetails(pattern=r"'", escape_symbols=r"\'"), + ], + ValueType.regex_value: [ + EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\\\\\"), + EscapeDetails(pattern=r'"', escape_symbols=r"\""), + EscapeDetails(pattern=r"'", escape_symbols=r"\'"), + ], + } + + +esql_query_escape_manager = ESQLQueryEscapeManager() diff --git a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py index b0489fbf..6c02cdbe 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/mapping.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/mapping.py @@ -1,12 +1,15 @@ from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import ( elastalert_details, + elasticsearch_esql_query_details, elasticsearch_lucene_query_details, elasticsearch_rule_details, kibana_rule_details, xpack_watcher_details, ) +DEFAULT_MAPPING_NAME = "default" + elasticsearch_lucene_query_mappings = LuceneMappings( platform_dir="elasticsearch", platform_details=elasticsearch_lucene_query_details ) @@ -14,3 +17,13 @@ elastalert_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=elastalert_details) kibana_rule_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=kibana_rule_details) xpack_watcher_mappings = LuceneMappings(platform_dir="elasticsearch", platform_details=xpack_watcher_details) + + +class ElasticESQLMappings(LuceneMappings): + is_strict_mapping: bool = True + skip_load_default_mappings = True + + +esql_query_mappings = ElasticESQLMappings( + platform_dir="elasticsearch_esql", platform_details=elasticsearch_esql_query_details +) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py index 7e64eea6..1142a26d 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -24,7 +24,7 @@ from app.translator.core.mapping import SourceMapping from app.translator.core.mitre import MitreConfig, MitreInfoContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.models.query_container import MetaInfoContainer +from app.translator.core.models.query_container import MetaInfoContainer, MitreTechniqueContainer from app.translator.managers import render_manager from app.translator.platforms.base.lucene.mapping import LuceneMappings from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_DETECTION_RULE, elasticsearch_rule_details @@ -66,8 +66,8 @@ def __create_mitre_threat(self, mitre_attack: MitreInfoContainer) -> Union[list, technique_id = technique.technique_id.lower() if "." in technique_id: technique_id = technique_id[: technique.technique_id.index(".")] - main_technique = self.mitre.get_technique(technique_id) - if tactic.name in main_technique.tactic: + main_technique: Optional[MitreTechniqueContainer] = self.mitre.techniques.search(technique_id) + if main_technique and tactic.name in main_technique.tactic: sub_threat["technique"].append( { "id": main_technique.technique_id, diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py new file mode 100644 index 00000000..9882e4e3 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql.py @@ -0,0 +1,139 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from typing import Optional, Union + +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.custom_types.values import ValueType +from app.translator.core.exceptions.render import UnsupportedRenderMethod +from app.translator.core.mapping import LogSourceSignature +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.managers import render_manager +from app.translator.platforms.elasticsearch.const import elasticsearch_esql_query_details +from app.translator.platforms.elasticsearch.mapping import ElasticESQLMappings, esql_query_mappings +from app.translator.platforms.elasticsearch.str_value_manager import ( + ESQLQueryStrValueManager, + esql_query_str_value_manager, +) + + +class ESQLFieldValueRender(BaseFieldValueRender): + details: PlatformDetails = elasticsearch_esql_query_details + str_value_manager: ESQLQueryStrValueManager = esql_query_str_value_manager + + @staticmethod + def _make_case_insensitive(value: str) -> str: + container: list[str] = [] + for v in value: + if v.isalpha(): + container.append(f"[{v.upper()}{v.lower()}]") + else: + container.append(v) + return "".join(container) + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f'"{value}"' + + @staticmethod + def _wrap_int_value(value: int) -> str: + return f'"{value}"' + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} == {value}" + + def less_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} < {value}" + + def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} <= {value}" + + def greater_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} > {value}" + + def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} >= {value}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} != {value}" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})" + if field.endswith(".text"): + return self.regex_modifier(field=field, value=value) + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=False, wrap_int=True) + return f'{field} like "*{value}*"' + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field.endswith(".text"): + return self.regex_modifier(field=field, value=value) + if isinstance(value, list): + return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"ends_with({field}, {value})" + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if field.endswith(".text"): + return self.regex_modifier(field=field, value=value) + if isinstance(value, list): + return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"starts_with({field}, {value})" + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=False, wrap_int=True) + if isinstance(value, str): + value = self._make_case_insensitive(value) + return f'{field} rlike ".*{value}.*"' + + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 + raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords") + + +@render_manager.register +class ESQLQueryRender(PlatformQueryRender): + details: PlatformDetails = elasticsearch_esql_query_details + mappings: ElasticESQLMappings = esql_query_mappings + comment_symbol = "//" + + or_token = "or" + and_token = "and" + not_token = "not" + field_value_render = ESQLFieldValueRender(or_token=or_token) + + def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002 + table = str(log_source_signature) if str(log_source_signature) else "*" + return f"FROM {table} |" + + @staticmethod + def _finalize_search_query(query: str) -> str: + return f"WHERE {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py new file mode 100644 index 00000000..6eebf0c4 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/esql_rule.py @@ -0,0 +1,115 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +import copy +import json +from typing import Optional, Union + +from app.translator.core.mapping import LogSourceSignature, SourceMapping +from app.translator.core.mitre import MitreConfig +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer, MitreInfoContainer +from app.translator.managers import render_manager +from app.translator.platforms.elasticsearch.const import ESQL_RULE, elasticsearch_esql_rule_details +from app.translator.platforms.elasticsearch.mapping import LuceneMappings, esql_query_mappings +from app.translator.platforms.elasticsearch.renders.esql import ESQLFieldValueRender, ESQLQueryRender + +_AUTOGENERATED_TEMPLATE = "Autogenerated ESQL Rule" + + +class ESQLRuleFieldValueRender(ESQLFieldValueRender): + details: PlatformDetails = elasticsearch_esql_rule_details + + +@render_manager.register +class ESQLRuleRender(ESQLQueryRender): + details: PlatformDetails = elasticsearch_esql_rule_details + mappings: LuceneMappings = esql_query_mappings + mitre: MitreConfig = MitreConfig() + + or_token = "or" + field_value_render = ESQLRuleFieldValueRender(or_token=or_token) + + def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002 + table = str(log_source_signature) if str(log_source_signature) else "*" + return f"FROM {table} metadata _id, _version, _index |" + + def __create_mitre_threat(self, mitre_attack: MitreInfoContainer) -> Union[list, list[dict]]: + if not mitre_attack.techniques: + return [] + threat = [] + + for tactic in mitre_attack.tactics: + tactic_render = {"id": tactic.external_id, "name": tactic.name, "reference": tactic.url} + sub_threat = {"tactic": tactic_render, "framework": "MITRE ATT&CK", "technique": []} + for technique in mitre_attack.techniques: + technique_id = technique.technique_id.lower() + if "." in technique_id: + technique_id = technique_id[: technique.technique_id.index(".")] + main_technique = self.mitre.get_technique(technique_id) + if main_technique and tactic.name in main_technique.tactic: + sub_threat["technique"].append( + { + "id": main_technique.technique_id, + "name": main_technique.name, + "reference": main_technique.url, + } + ) + if len(sub_threat["technique"]) > 0: + threat.append(sub_threat) + + return sorted(threat, key=lambda x: x["tactic"]["id"]) + + def finalize_query( + self, + prefix: str, + query: str, + functions: str, + meta_info: Optional[MetaInfoContainer] = None, + source_mapping: Optional[SourceMapping] = None, # noqa: ARG002 + not_supported_functions: Optional[list] = None, + *args, # noqa: ARG002 + **kwargs, # noqa: ARG002 + ) -> str: + query = super().finalize_query(prefix=prefix, query=query, functions=functions) + rule = copy.deepcopy(ESQL_RULE) + rule.update( + { + "query": query, + "description": meta_info.description if meta_info else rule["description"] or _AUTOGENERATED_TEMPLATE, + "name": meta_info.title if meta_info else _AUTOGENERATED_TEMPLATE, + } + ) + if meta_info: + rule.update( + { + "rule_id": meta_info.id, + "author": [meta_info.author], + "severity": meta_info.severity, + "references": meta_info.references, + "license": meta_info.license, + "tags": meta_info.tags, + "threat": self.__create_mitre_threat(meta_info.mitre_attack), + "false_positives": meta_info.false_positives, + } + ) + rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) + if not_supported_functions: + rendered_not_supported = self.render_not_supported_functions(not_supported_functions) + return rule_str + rendered_not_supported + return rule_str diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py new file mode 100644 index 00000000..e1b8708a --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -0,0 +1,40 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from typing import ClassVar + +from app.translator.core.str_value_manager import ( + BaseSpecSymbol, + ReDigitalSymbol, + ReWhiteSpaceSymbol, + ReWordSymbol, + StrValueManager, +) +from app.translator.platforms.elasticsearch.escape_manager import ESQLQueryEscapeManager, esql_query_escape_manager + + +class ESQLQueryStrValueManager(StrValueManager): + escape_manager: ESQLQueryEscapeManager = esql_query_escape_manager + re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { + "w": ReWordSymbol, + "d": ReDigitalSymbol, + "s": ReWhiteSpaceSymbol, + } + + +esql_query_str_value_manager = ESQLQueryStrValueManager() diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index ef914245..138e56c6 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -22,7 +22,6 @@ from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType -from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import SourceMapping from app.translator.core.mitre import MitreInfoContainer from app.translator.core.models.platform_details import PlatformDetails @@ -167,21 +166,6 @@ def not_regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: value = self.__prepare_regex_value(value) return f'{field} NOT REGEXP "{value}"' - def less_modifier(self, field: str, value: Union[int, str]) -> str: # noqa: ARG002 - raise UnsupportedRenderMethod(platform_name=self.details.name, method="<") - - def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: # noqa: ARG002 - raise UnsupportedRenderMethod(platform_name=self.details.name, method="<=") - - def greater_modifier(self, field: str, value: Union[int, str]) -> str: # noqa: ARG002 - raise UnsupportedRenderMethod(platform_name=self.details.name, method=">") - - def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: # noqa: ARG002 - raise UnsupportedRenderMethod(platform_name=self.details.name, method=">=") - - def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 - raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords") - @render_manager.register class FortiSiemRuleRender(PlatformQueryRender): diff --git a/uncoder-core/app/translator/platforms/microsoft/__init__.py b/uncoder-core/app/translator/platforms/microsoft/__init__.py index 623fe77a..45fa896b 100644 --- a/uncoder-core/app/translator/platforms/microsoft/__init__.py +++ b/uncoder-core/app/translator/platforms/microsoft/__init__.py @@ -1,6 +1,8 @@ from app.translator.platforms.microsoft.parsers.microsoft_defender import MicrosoftDefenderQueryParser # noqa: F401 from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser # noqa: F401 -from app.translator.platforms.microsoft.parsers.microsoft_sentinel_rule import MicrosoftSentinelRuleParser # noqa: F401 +from app.translator.platforms.microsoft.parsers.microsoft_sentinel_rule import ( + MicrosoftSentinelRuleParser, # noqa: F401 +) from app.translator.platforms.microsoft.renders.microsoft_defender import MicrosoftDefenderQueryRender # noqa: F401 from app.translator.platforms.microsoft.renders.microsoft_defender_cti import MicrosoftDefenderCTI # noqa: F401 from app.translator.platforms.microsoft.renders.microsoft_sentinel import MicrosoftSentinelQueryRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/microsoft/const.py b/uncoder-core/app/translator/platforms/microsoft/const.py index 02a2a7d0..5a877d8a 100644 --- a/uncoder-core/app/translator/platforms/microsoft/const.py +++ b/uncoder-core/app/translator/platforms/microsoft/const.py @@ -34,6 +34,14 @@ **PLATFORM_DETAILS, } +MICROSOFT_SENTINEL_YAML_RULE_DETAILS = { + "platform_id": "sentinel-kql-yaml-rule", + "name": "Microsoft Sentinel YAML Rule", + "platform_name": "YAML Rule (Kusto)", + "first_choice": 0, + **PLATFORM_DETAILS, +} + MICROSOFT_DEFENDER_DETAILS = { "platform_id": "mde-kql-query", "group_name": "Microsoft Defender for Endpoint", @@ -45,3 +53,4 @@ microsoft_defender_query_details = PlatformDetails(**MICROSOFT_DEFENDER_DETAILS) microsoft_sentinel_query_details = PlatformDetails(**MICROSOFT_SENTINEL_QUERY_DETAILS) microsoft_sentinel_rule_details = PlatformDetails(**MICROSOFT_SENTINEL_RULE_DETAILS) +microsoft_sentinel_yaml_rule_details = PlatformDetails(**MICROSOFT_SENTINEL_YAML_RULE_DETAILS) diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py index f168798e..4a2cf6bf 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py @@ -18,16 +18,24 @@ from contextlib import suppress from datetime import timedelta -from typing import Optional +from typing import Optional, Union import isodate from isodate.isoerror import ISO8601Error -from app.translator.core.mixins.rule import JsonRuleMixin +from app.translator.core.mixins.rule import JsonRuleMixin, YamlRuleMixin from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer +from app.translator.core.models.query_container import ( + MetaInfoContainer, + MitreInfoContainer, + RawMetaInfoContainer, + RawQueryContainer, +) from app.translator.managers import parser_manager -from app.translator.platforms.microsoft.const import microsoft_sentinel_rule_details +from app.translator.platforms.microsoft.const import ( + microsoft_sentinel_rule_details, + microsoft_sentinel_yaml_rule_details, +) from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_rule_mappings from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser from app.translator.tools.utils import parse_rule_description_str @@ -39,7 +47,7 @@ class MicrosoftSentinelRuleParser(MicrosoftSentinelQueryParser, JsonRuleMixin): mappings: MicrosoftSentinelMappings = microsoft_sentinel_rule_mappings @staticmethod - def __parse_timeframe(raw_timeframe: Optional[str]) -> Optional[timedelta]: + def _parse_timeframe(raw_timeframe: Optional[str]) -> Optional[timedelta]: with suppress(ISO8601Error): return isodate.parse_duration(raw_timeframe) @@ -65,7 +73,7 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: id_=parsed_description.get("rule_id"), title=rule.get("displayName"), description=parsed_description.get("description") or rule.get("description"), - timeframe=self.__parse_timeframe(rule.get("queryFrequency", "")), + timeframe=self._parse_timeframe(rule.get("queryFrequency", "")), severity=rule.get("severity", "medium"), mitre_attack=mitre_attack, author=parsed_description.get("author") or [rule.get("author")], @@ -74,3 +82,78 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: references=parsed_description.get("references"), ), ) + + +@parser_manager.register +class MicrosoftSentinelYAMLRuleParser(YamlRuleMixin, MicrosoftSentinelRuleParser): + details: PlatformDetails = microsoft_sentinel_yaml_rule_details + mappings: MicrosoftSentinelMappings = microsoft_sentinel_rule_mappings + + def extract_tags(self, data: Union[dict, list, str]) -> list[str]: + tags = [] + if isinstance(data, dict): + for key, value in data.items(): + tags.extend(self.extract_tags(value)) + elif isinstance(data, list): + for item in data: + tags.extend(self.extract_tags(item)) + elif isinstance(data, str): + tags.append(data) + return tags + + def __get_tags_from_required_data_connectors(self, required_data_connectors: dict) -> list[str]: + return list(self.extract_tags(required_data_connectors)) + + def __get_tags_from_metadata(self, metadata: dict) -> list[str]: + fields_to_process = {} + for k, v in metadata.items(): + if k.lower() != "author": + fields_to_process[k] = v + + return list(self.extract_tags(fields_to_process)) + + def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: + rule = self.load_rule(text=text) + tags = [] + mitre_attack: MitreInfoContainer = self.mitre_config.get_mitre_info( + tactics=[tactic.lower() for tactic in rule.get("tactics", [])], + techniques=[technique.lower() for technique in rule.get("relevantTechniques", [])], + ) + + if mitre_attack: + for technique in mitre_attack.techniques: + tags.append(technique.technique_id.lower()) + for tactic in mitre_attack.tactics: + tags.append(tactic.name.lower().replace(" ", "_")) + + tags.extend(self.__get_tags_from_required_data_connectors(rule.get("requiredDataConnectors", {}))) + tags.extend(self.__get_tags_from_metadata(rule.get("metadata", {}))) + + for tag in rule.get("tags", []): + if isinstance(tag, str): + tags.append(tag) + + timeframe = self._parse_timeframe(rule.get("queryFrequency", "")) + query_period = self._parse_timeframe(rule.get("queryPeriod", "")) + + return RawQueryContainer( + query=rule["query"], + language=language, + meta_info=MetaInfoContainer( + id_=rule.get("id"), + title=rule.get("name"), + description=rule.get("description"), + timeframe=timeframe, + query_period=query_period, + severity=rule.get("severity", "medium").lower(), + mitre_attack=mitre_attack, + author=rule.get("metadata", {}).get("author", {}).get("name", "").split(","), + tags=sorted(set(tags)), + raw_metainfo_container=RawMetaInfoContainer( + trigger_operator=rule.get("triggerOperator"), + trigger_threshold=rule.get("triggerThreshold"), + query_frequency=rule.get("queryFrequency"), + query_period=rule.get("queryPeriod"), + ), + ), + ) diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index e689ee0b..11722f89 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -21,6 +21,8 @@ import json from typing import Optional +import isodate + from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import SourceMapping from app.translator.core.models.platform_details import PlatformDetails @@ -69,6 +71,30 @@ def __create_mitre_threat(self, mitre_attack: MitreInfoContainer) -> tuple[list, return sorted(tactics), sorted(techniques) + @staticmethod + def get_query_frequency(meta_info: MetaInfoContainer) -> Optional[str]: + if meta_info.timeframe: + return isodate.duration_isoformat(meta_info.timeframe) + if meta_info.raw_metainfo_container: + return meta_info.raw_metainfo_container.query_frequency + + @staticmethod + def get_query_period(meta_info: MetaInfoContainer) -> Optional[str]: + if meta_info.query_period: + return isodate.duration_isoformat(meta_info.query_period) + if meta_info.raw_metainfo_container: + return meta_info.raw_metainfo_container.query_period + + @staticmethod + def get_trigger_operator(meta_info: MetaInfoContainer) -> Optional[str]: + if meta_info.raw_metainfo_container: + return meta_info.raw_metainfo_container.trigger_operator + + @staticmethod + def get_trigger_threshold(meta_info: MetaInfoContainer) -> Optional[str]: + if meta_info.raw_metainfo_container: + return meta_info.raw_metainfo_container.trigger_threshold + def finalize_query( self, prefix: str, @@ -94,6 +120,13 @@ def finalize_query( mitre_tactics, mitre_techniques = self.__create_mitre_threat(mitre_attack=meta_info.mitre_attack) rule["tactics"] = mitre_tactics rule["techniques"] = mitre_techniques + + if meta_info: + rule["queryFrequency"] = self.get_query_frequency(meta_info=meta_info) or rule["queryFrequency"] + rule["queryPeriod"] = self.get_query_period(meta_info=meta_info) or rule["queryPeriod"] + rule["triggerOperator"] = self.get_trigger_operator(meta_info=meta_info) or rule["triggerOperator"] + rule["triggerThreshold"] = self.get_trigger_threshold(meta_info=meta_info) or rule["triggerThreshold"] + json_rule = json.dumps(rule, indent=4, sort_keys=False) json_rule = self.wrap_with_unmapped_fields(json_rule, unmapped_fields) return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) From 6fd6ef78ded259032ad5dab2e8e928a3c1d3467f Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:27:52 +0300 Subject: [PATCH 09/13] gis-8502 fix MetaInfoContainer --- .../translator/core/models/query_container.py | 16 ++++++++-------- .../platforms/elasticsearch/__init__.py | 2 +- .../elasticsearch/parsers/detection_rule.py | 5 ++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index 76159a4c..372e98fb 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -39,11 +39,15 @@ def __init__( trigger_threshold: Optional[str] = None, query_frequency: Optional[str] = None, query_period: Optional[str] = None, + from_: Optional[str] = None, + interval: Optional[str] = None, ) -> None: self.trigger_operator = trigger_operator self.trigger_threshold = trigger_threshold self.query_frequency = query_frequency self.query_period = query_period + self.from_ = from_ + self.interval = interval class MetaInfoContainer: @@ -51,12 +55,10 @@ def __init__( self, *, id_: Optional[str] = None, - from_: Optional[str] = None, - index: Optional[str] = None, + index: Optional[list[str]] = None, language: Optional[str] = None, - risk_score: Optional[str] = None, + risk_score: Optional[int] = None, type_: Optional[str] = None, - interval: Optional[str] = None, title: Optional[str] = None, description: Optional[str] = None, author: Optional[list[str]] = None, @@ -79,12 +81,10 @@ def __init__( ) -> None: self.id = id_ or str(uuid.uuid4()) self.title = title or "" - self.from_ = from_ or "" - self.index = index or "" + self.index = index or [] self.language = language or "" - self.risk_score = risk_score or "" + self.risk_score = risk_score or None self.type_ = type_ or "" - self.interval = interval or "" self.description = description or "" self.author = [v.strip() for v in author] if author else [] self.date = date or datetime.now().date().strftime("%Y-%m-%d") diff --git a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py index e38205d8..91a7d362 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py @@ -1,5 +1,5 @@ from app.translator.platforms.elasticsearch.parsers.detection_rule import ( - ElasticSearchRuleParser, + ElasticSearchRuleParser, # noqa: F401 ElasticSearchRuleTOMLParser, # noqa: F401 ) from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py index 45630688..1142cd4b 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py @@ -19,7 +19,7 @@ from app.translator.core.mixins.rule import JsonRuleMixin, TOMLRuleMixin from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer +from app.translator.core.models.query_container import MetaInfoContainer, RawMetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager from app.translator.platforms.elasticsearch.const import elasticsearch_rule_details, elasticsearch_rule_toml_details from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser @@ -89,11 +89,10 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: references=rule.get("references"), tags=rule.get("tags"), mitre_attack=mitre_attack, - from_=rule.get("from"), index=rule.get("index"), language=rule.get("language"), risk_score=rule.get("risk_score"), type_=rule.get("type"), - interval=rule.get("interval"), + raw_metainfo_container=RawMetaInfoContainer(from_=rule.get("from"), interval=rule.get("interval")), ), ) From 4708fc5bf618116a64208b9151ba06a38e839a1b Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:31:58 +0300 Subject: [PATCH 10/13] gis-8502 fix MetaInfoContainer --- uncoder-core/app/translator/core/models/query_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index 372e98fb..bb95f9b4 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -83,7 +83,7 @@ def __init__( self.title = title or "" self.index = index or [] self.language = language or "" - self.risk_score = risk_score or None + self.risk_score = risk_score self.type_ = type_ or "" self.description = description or "" self.author = [v.strip() for v in author] if author else [] From 9c573670dfa271585a48cbbb58cab9d55525d0d7 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:35:34 +0300 Subject: [PATCH 11/13] gis-8502 fix --- .../platforms/elasticsearch/parsers/detection_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py index 1142cd4b..67f95662 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py @@ -66,7 +66,7 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: metadata = raw_rule.get("metadata") techniques = [] for threat_data in rule.get("threat", []): - if threat_data.get("technique") and len(threat_data.get("technique")) > 0: + if len(threat_data.get("technique", [])) > 0: techniques.append(threat_data["technique"][0]["id"].lower()) mitre_attack = self.mitre_config.get_mitre_info( tactics=[threat_data["tactic"]["name"].replace(" ", "_").lower() for threat_data in rule.get("threat", [])], From f80fb43f469ba569aeb6a57ef2130fe489de63ec Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:37:48 +0300 Subject: [PATCH 12/13] gis-8502 fix From b3c577b4b032eff86d07241fa39a9628377d0e4b Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:37:52 +0300 Subject: [PATCH 13/13] gis-8502 fix --- .../platforms/elasticsearch/parsers/detection_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py index 67f95662..6d04b229 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/detection_rule.py @@ -66,7 +66,7 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: metadata = raw_rule.get("metadata") techniques = [] for threat_data in rule.get("threat", []): - if len(threat_data.get("technique", [])) > 0: + if threat_data.get("technique"): techniques.append(threat_data["technique"][0]["id"].lower()) mitre_attack = self.mitre_config.get_mitre_info( tactics=[threat_data["tactic"]["name"].replace(" ", "_").lower() for threat_data in rule.get("threat", [])],
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies: