From e55a7dfd18d9409a61b194428382f273f418460c Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:28:57 +0300 Subject: [PATCH 1/9] gis-83825 change sentinel one const --- .../translator/platforms/sentinel_one/const.py | 17 +++++++++++++++-- .../platforms/sentinel_one/renders/s1_cti.py | 4 ++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/uncoder-core/app/translator/platforms/sentinel_one/const.py b/uncoder-core/app/translator/platforms/sentinel_one/const.py index b9dc9dbe..c53b8d75 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/const.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/const.py @@ -1,7 +1,20 @@ +from app.translator.core.models.platform_details import PlatformDetails + + +PLATFORM_DETAILS = {"group_id": "sentinel-one", "group_name": "SentinelOne"} + SENTINEL_ONE_EVENTS_QUERY_DETAILS = { "platform_id": "s1-events", "name": "SentinelOne Events Query", - "group_name": "SentinelOne", - "group_id": "sentinel-one", "platform_name": "Query (Events)", + **PLATFORM_DETAILS, } + +SENTINEL_ONE_POWER_QUERY_DETAILS = { + "platform_id": "sentinel-one-power-query", + "name": "SentinelOne Power Query", + "platform_name": "Power Query", + **PLATFORM_DETAILS, +} + +sentinel_one_events_query_details = PlatformDetails(**SENTINEL_ONE_EVENTS_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/sentinel_one/renders/s1_cti.py b/uncoder-core/app/translator/platforms/sentinel_one/renders/s1_cti.py index 917ec84c..8c416a1d 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/renders/s1_cti.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/renders/s1_cti.py @@ -20,13 +20,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render_cti import RenderCTI from app.translator.managers import render_cti_manager -from app.translator.platforms.sentinel_one.const import SENTINEL_ONE_EVENTS_QUERY_DETAILS +from app.translator.platforms.sentinel_one.const import sentinel_one_events_query_details from app.translator.platforms.sentinel_one.mappings.s1_cti import DEFAULT_S1EVENTS_MAPPING @render_cti_manager.register class S1EventsCTI(RenderCTI): - details: PlatformDetails = PlatformDetails(**SENTINEL_ONE_EVENTS_QUERY_DETAILS) + details: PlatformDetails = sentinel_one_events_query_details field_value_template: str = '"{value}"' or_operator: str = ", " From 99cfbba2fb27a6e47d920241893ee75cec54012c Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Fri, 11 Oct 2024 09:36:05 +0300 Subject: [PATCH 2/9] gis-8825 added sentinel one power query mappings --- .../platforms/sentinel_one/default.yml | 2 ++ .../mappings/platforms/sentinel_one/dns.yml | 12 +++++++++++ .../sentinel_one/linux_file_event.yml | 11 ++++++++++ .../sentinel_one/windows_image_load.yml | 9 ++++++++ .../windows_network_connection.yml | 21 +++++++++++++++++++ .../sentinel_one/windows_pipe_created.yml | 9 ++++++++ .../sentinel_one/windows_process_creation.yml | 21 +++++++++++++++++++ .../sentinel_one/windows_registry_event.yml | 10 +++++++++ .../platforms/sentinel_one/const.py | 1 + 9 files changed, 96 insertions(+) create mode 100644 uncoder-core/app/translator/mappings/platforms/sentinel_one/default.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/sentinel_one/dns.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/sentinel_one/linux_file_event.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_image_load.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_network_connection.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_pipe_created.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_process_creation.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_registry_event.yml diff --git a/uncoder-core/app/translator/mappings/platforms/sentinel_one/default.yml b/uncoder-core/app/translator/mappings/platforms/sentinel_one/default.yml new file mode 100644 index 00000000..16f2f7e5 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/sentinel_one/default.yml @@ -0,0 +1,2 @@ +platform: Sentinel One Power Query +source: default diff --git a/uncoder-core/app/translator/mappings/platforms/sentinel_one/dns.yml b/uncoder-core/app/translator/mappings/platforms/sentinel_one/dns.yml new file mode 100644 index 00000000..d04d59e2 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/sentinel_one/dns.yml @@ -0,0 +1,12 @@ +platform: Sentinel One Power Query +source: dns + +field_mapping: + Image: src.process.image.path + CommandLine: src.process.cmdline + ParentImage: src.process.parent.image.path + ParentCommandLine: src.process.parent.cmdline + query: event.dns.request + answer: event.dns.response + QueryName: event.dns.request + record_type: event.dns.response \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/sentinel_one/linux_file_event.yml b/uncoder-core/app/translator/mappings/platforms/sentinel_one/linux_file_event.yml new file mode 100644 index 00000000..7fbf5c59 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/sentinel_one/linux_file_event.yml @@ -0,0 +1,11 @@ +platform: Sentinel One Power Query +source: linux_file_event + +field_mapping: + Image: src.process.image.path + CommandLine: src.process.cmdline + ParentImage: src.process.parent.image.path + ParentCommandLine: src.process.parent.cmdline + TargetFilename: tgt.file.path + SourceFilename: tgt.file.oldPath + User: src.process.use \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_image_load.yml new file mode 100644 index 00000000..e6533f12 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_image_load.yml @@ -0,0 +1,9 @@ +platform: Sentinel One Power Query +source: windows_image_load + +field_mapping: + Image: Image + ImageLoaded: ImageLoaded + SignatureStatus: SignatureStatus + OriginalFileName: OriginalFileName + Signed: Signed \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_network_connection.yml new file mode 100644 index 00000000..f740d589 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_network_connection.yml @@ -0,0 +1,21 @@ +platform: Sentinel One Power Query +source: windows_network_connection + +field_mapping: + Image: src.process.image.path + CommandLine: src.process.cmdline + ParentImage: src.process.parent.image.path + ParentCommandLine: src.process.parent.cmdline + DestinationHostname: + - url.address + - event.dns.request + DestinationPort: dst.port.number + DestinationIp: dst.ip.address + User: src.process.user + SourceIp: src.ip.address + SourcePort: src.port.number + Protocol: NetProtocolName + dst_ip: dst.ip.address + src_ip: src.ip.address + dst_port: dst.port.number + src_port: src.port.number \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_pipe_created.yml b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_pipe_created.yml new file mode 100644 index 00000000..0f670481 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_pipe_created.yml @@ -0,0 +1,9 @@ +platform: Sentinel One Power Query +source: windows_pipe_created + +field_mapping: + PipeName: namedPipe.name + Image: src.process.image.path + CommandLine: src.process.cmdline + ParentImage: src.process.parent.image.path + ParentCommandLine: src.process.parent.cmdline \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_process_creation.yml new file mode 100644 index 00000000..f736fa46 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_process_creation.yml @@ -0,0 +1,21 @@ +platform: Sentinel One Power Query +source: windows_process_creation + +field_mapping: + ProcessId: tgt.process.pid + Image: tgt.process.image.path + Description: tgt.process.displayName + Publisher: tgt.process.publisher + Product: tgt.process.displayName + Company: tgt.process.publisher + CommandLine: tgt.process.cmdline + CurrentDirectory: tgt.process.image.path + User: tgt.process.user + TerminalSessionId: tgt.process.sessionid + IntegrityLevel: tgt.process.integrityLevel + md5: tgt.process.image.md5 + sha1: tgt.process.image.sha1 + sha256: tgt.process.image.sha256 + ParentProcessId: src.process.pid + ParentImage: src.process.image.path + ParentCommandLine: src.process.cmdline \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_registry_event.yml b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_registry_event.yml new file mode 100644 index 00000000..72f8db79 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/sentinel_one/windows_registry_event.yml @@ -0,0 +1,10 @@ +platform: Sentinel One Power Query +source: windows_registry_event + +field_mapping: + Image: src.process.image.path + CommandLine: src.process.cmdline + ParentImage: src.process.parent.image.path + ParentCommandLine: src.process.parent.cmdline + TargetObject: registry.keyPath + Details: registry.value \ No newline at end of file diff --git a/uncoder-core/app/translator/platforms/sentinel_one/const.py b/uncoder-core/app/translator/platforms/sentinel_one/const.py index c53b8d75..4e1b6d65 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/const.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/const.py @@ -18,3 +18,4 @@ } sentinel_one_events_query_details = PlatformDetails(**SENTINEL_ONE_EVENTS_QUERY_DETAILS) +sentinel_one_power_query_details = PlatformDetails(**SENTINEL_ONE_POWER_QUERY_DETAILS) From 9231d12f85e01901e74a208b4b26f3780ddf98f5 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:02:56 +0300 Subject: [PATCH 3/9] Merge branch 'prod' into gis-8825 --- uncoder-core/app/routers/meta_info.py | 88 +++++++++ uncoder-core/app/translator/core/mapping.py | 53 +++++- .../app/translator/core/mixins/rule.py | 2 +- .../translator/core/models/query_container.py | 10 +- uncoder-core/app/translator/core/parser.py | 25 ++- uncoder-core/app/translator/core/render.py | 8 +- .../platforms/carbonblack/default.yml | 2 + .../platforms/carbonblack/linux_dns_query.yml | 8 + .../carbonblack/linux_network_connection.yml | 9 + .../platforms/carbonblack/macos_dns_query.yml | 8 + .../carbonblack/macos_network_connection.yml | 9 + .../windows_create_remote_thread.yml | 7 + .../carbonblack/windows_dns_query.yml | 8 + .../carbonblack/windows_file_event.yml | 8 + .../carbonblack/windows_image_load.yml | 6 + .../windows_network_connection.yml | 9 + .../carbonblack/windows_process_creation.yml | 14 ++ .../carbonblack/windows_registry_event.yml | 9 + .../carbonblack/windows_security.yml | 17 ++ .../platforms/carbonblack/windows_sysmon.yml | 51 +++++ .../elasticsearch/windows_bits_client.yml | 2 + .../elasticsearch/windows_image_load.yml | 4 + .../elasticsearch/windows_ldap_debug.yml | 2 + .../windows_network_connection.yml | 2 + .../platforms/elasticsearch/windows_ntlm.yml | 2 + .../windows_process_creation.yml | 2 + .../elasticsearch/windows_security.yml | 2 + .../elasticsearch/windows_sysmon.yml | 2 + .../platforms/base/aql/parsers/aql.py | 16 +- .../platforms/base/lucene/parsers/lucene.py | 14 +- .../platforms/base/lucene/tokenizer.py | 7 +- .../platforms/base/spl/parsers/spl.py | 14 +- .../platforms/base/sql/parsers/sql.py | 16 +- .../platforms/carbonblack/__init__.py | 1 + .../translator/platforms/carbonblack/const.py | 4 + .../platforms/carbonblack/escape_manager.py | 20 ++ .../platforms/carbonblack/mapping.py | 18 ++ .../carbonblack/renders/carbonblack.py | 103 +++++++++++ .../carbonblack/renders/carbonblack_cti.py | 4 +- .../carbonblack/str_value_manager.py | 38 ++++ .../translator/platforms/chronicle/const.py | 6 +- .../platforms/chronicle/parsers/chronicle.py | 6 +- .../chronicle/parsers/chronicle_rule.py | 50 +++-- .../platforms/elasticsearch/__init__.py | 2 + .../platforms/elasticsearch/escape_manager.py | 8 + .../parsers/elasticsearch_eql.py | 37 ++++ .../renders/elasticsearch_eql.py | 174 ++++++++++++++++++ .../elasticsearch/str_value_manager.py | 27 ++- .../forti_siem/renders/forti_siem_rule.py | 5 +- .../renders/logrhythm_axon_query.py | 5 +- .../platforms/logscale/parsers/logscale.py | 10 +- .../microsoft/parsers/microsoft_sentinel.py | 14 +- 52 files changed, 889 insertions(+), 79 deletions(-) create mode 100644 uncoder-core/app/routers/meta_info.py create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/default.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/linux_dns_query.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/linux_network_connection.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/macos_dns_query.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/macos_network_connection.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/windows_create_remote_thread.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/windows_dns_query.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/windows_file_event.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/windows_image_load.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/windows_network_connection.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/windows_process_creation.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/windows_registry_event.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/windows_security.yml create mode 100644 uncoder-core/app/translator/mappings/platforms/carbonblack/windows_sysmon.yml create mode 100644 uncoder-core/app/translator/platforms/carbonblack/escape_manager.py create mode 100644 uncoder-core/app/translator/platforms/carbonblack/mapping.py create mode 100644 uncoder-core/app/translator/platforms/carbonblack/renders/carbonblack.py create mode 100644 uncoder-core/app/translator/platforms/carbonblack/str_value_manager.py create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py create mode 100644 uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py diff --git a/uncoder-core/app/routers/meta_info.py b/uncoder-core/app/routers/meta_info.py new file mode 100644 index 00000000..7bf6eb60 --- /dev/null +++ b/uncoder-core/app/routers/meta_info.py @@ -0,0 +1,88 @@ +from dataclasses import asdict + +from fastapi import APIRouter, Body, HTTPException + +from app.models.meta_info import ( + MetaInfo, + MetaInfoResponse, + MitreInfoContainer, + MitreTacticContainer, + MitreTechniqueContainer, + ParsedLogSources, + RawMetaInfo, +) +from app.translator.core.exceptions.core import UnsupportedPlatform +from app.translator.translator import app_translator + +meta_info_router = APIRouter() + + +@meta_info_router.post("/get_meta_info/", tags=["meta_info"], description="Get Rule MetaInfo") +@meta_info_router.post("/get_meta_info/", include_in_schema=False) +def get_meta_info_data( + source_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True) +) -> MetaInfoResponse: + try: + logsources, raw_query_container = app_translator.parse_meta_info(text=text, source=source_platform_id) + except UnsupportedPlatform as exc: + raise HTTPException(status_code=400, detail="Unsuported platform") from exc + except Exception as exc: + raise HTTPException(status_code=400, detail="Unexpected error.") from exc + if not raw_query_container: + raise HTTPException(status_code=400, detail="Can't parse metadata") + most_frequent_product = max(logsources.get("product"), key=logsources.get("product").get, default=None) + most_frequent_service = max(logsources.get("service"), key=logsources.get("service").get, default=None) + most_frequent_category = max(logsources.get("category"), key=logsources.get("category").get, default=None) + + logsources.get("product", {}).pop(most_frequent_product, None) + logsources.get("service", {}).pop(most_frequent_service, None) + logsources.get("category", {}).pop(most_frequent_category, None) + + parsed_logsources = ParsedLogSources( + most_frequent_product=most_frequent_product, + most_frequent_service=most_frequent_service, + most_frequent_category=most_frequent_category, + least_frequent_products=list(logsources.get("product", {}).keys()), + least_frequent_services=list(logsources.get("service", {}).keys()), + least_frequent_categories=list(logsources.get("category", {}).keys()), + ) + return MetaInfoResponse( + query=raw_query_container.query, + language=raw_query_container.language, + meta_info=MetaInfo( + id_=raw_query_container.meta_info.id, + title=raw_query_container.meta_info.title, + description=raw_query_container.meta_info.description, + author=raw_query_container.meta_info.author, + date=raw_query_container.meta_info.date, + false_positives=raw_query_container.meta_info.false_positives, + license_=raw_query_container.meta_info.license, + mitre_attack=MitreInfoContainer( + tactics=[ + MitreTacticContainer(**asdict(tactic_container)) + for tactic_container in raw_query_container.meta_info.mitre_attack.tactics + ], + techniques=[ + MitreTechniqueContainer(**asdict(tactic_container)) + for tactic_container in raw_query_container.meta_info.mitre_attack.techniques + ], + ), + output_table_fields=raw_query_container.meta_info.output_table_fields, + parsed_log_sources=parsed_logsources, + query_fields=raw_query_container.meta_info.query_fields + raw_query_container.meta_info.function_fields, + query_period=raw_query_container.meta_info.query_period, + raw_metainfo_container=RawMetaInfo( + trigger_operator=raw_query_container.meta_info.raw_metainfo_container.trigger_operator, + trigger_threshold=raw_query_container.meta_info.raw_metainfo_container.trigger_threshold, + query_frequency=raw_query_container.meta_info.raw_metainfo_container.query_frequency, + query_period=raw_query_container.meta_info.raw_metainfo_container.query_period, + ), + raw_mitre_attack=raw_query_container.meta_info.raw_mitre_attack, + references=raw_query_container.meta_info.references, + severity=raw_query_container.meta_info.severity, + source_mapping_ids=raw_query_container.meta_info.source_mapping_ids, + status=raw_query_container.meta_info.status, + tags=raw_query_container.meta_info.tags, + timeframe=raw_query_container.meta_info.timeframe, + ), + ) diff --git a/uncoder-core/app/translator/core/mapping.py b/uncoder-core/app/translator/core/mapping.py index 2a06147d..1c4d2070 100644 --- a/uncoder-core/app/translator/core/mapping.py +++ b/uncoder-core/app/translator/core/mapping.py @@ -22,6 +22,12 @@ class LogSourceSignature(ABC): def is_suitable(self, **kwargs) -> bool: raise NotImplementedError("Abstract method") + def is_probably_suitable(self, **kwargs) -> bool: + """ + Performs check with more options, but the result is less accurate than the "is_suitable" method + """ + raise NotImplementedError("Abstract method") + @staticmethod def _check_conditions(conditions: list[Union[bool, None]]) -> bool: conditions = [condition for condition in conditions if condition is not None] @@ -88,11 +94,13 @@ def __init__( log_source_signature: _LogSourceSignatureType = None, fields_mapping: Optional[FieldsMapping] = None, raw_log_fields: Optional[dict] = None, + conditions: Optional[dict] = None, ): self.source_id = source_id self.log_source_signature = log_source_signature self.fields_mapping = fields_mapping or FieldsMapping([]) self.raw_log_fields = raw_log_fields + self.conditions = conditions class BasePlatformMappings: @@ -123,6 +131,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]: field_mappings_dict = mapping_dict.get("field_mapping", {}) raw_log_fields = mapping_dict.get("raw_log_fields", {}) + conditions = mapping_dict.get("conditions", {}) field_mappings_dict.update({field: field for field in raw_log_fields}) fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict) self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping) @@ -131,6 +140,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]: log_source_signature=log_source_signature, fields_mapping=fields_mapping, raw_log_fields=raw_log_fields, + conditions=conditions, ) if self.skip_load_default_mappings: @@ -170,31 +180,47 @@ def get_source_mappings_by_fields_and_log_sources( return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]] - def get_source_mappings_by_ids(self, source_mapping_ids: list[str]) -> list[SourceMapping]: + def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]: + return self._source_mappings.get(source_id) + + def get_source_mappings_by_ids( + self, source_mapping_ids: list[str], return_default: bool = True + ) -> list[SourceMapping]: source_mappings = [] for source_mapping_id in source_mapping_ids: + if source_mapping_id == DEFAULT_MAPPING_NAME: + continue if source_mapping := self.get_source_mapping(source_mapping_id): source_mappings.append(source_mapping) - if not source_mappings: + if not source_mappings and return_default: source_mappings = [self.get_source_mapping(DEFAULT_MAPPING_NAME)] return source_mappings - def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]: - return self._source_mappings.get(source_id) + def get_source_mappings_by_log_sources(self, log_sources: dict) -> Optional[list[str]]: + raise NotImplementedError("Abstract method") @property def default_mapping(self) -> SourceMapping: return self._source_mappings[DEFAULT_MAPPING_NAME] - def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]: + def check_fields_mapping_existence( + self, + query_field_tokens: list[Field], + function_field_tokens_map: dict[str, list[Field]], + supported_func_render_names: set[str], + source_mapping: SourceMapping, + ) -> list[str]: unmapped = [] - for field in field_tokens: - generic_field_name = field.get_generic_field_name(source_mapping.source_id) - mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name) - if not mapped_field and field.source_name not in unmapped: - unmapped.append(field.source_name) + + for field in query_field_tokens: + self._check_field_mapping_existence(field, source_mapping, unmapped) + + for func_name, function_field_tokens in function_field_tokens_map.items(): + if func_name in supported_func_render_names: + for field in function_field_tokens: + self._check_field_mapping_existence(field, source_mapping, unmapped) if self.is_strict_mapping and unmapped: raise StrictPlatformException( @@ -203,6 +229,13 @@ def check_fields_mapping_existence(self, field_tokens: list[Field], source_mappi return unmapped + @staticmethod + def _check_field_mapping_existence(field: Field, source_mapping: SourceMapping, unmapped: list[str]) -> None: + generic_field_name = field.get_generic_field_name(source_mapping.source_id) + mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name) + if not mapped_field and field.source_name not in unmapped: + unmapped.append(field.source_name) + @staticmethod def map_field(field: Field, source_mapping: SourceMapping) -> list[str]: generic_field_name = field.get_generic_field_name(source_mapping.source_id) diff --git a/uncoder-core/app/translator/core/mixins/rule.py b/uncoder-core/app/translator/core/mixins/rule.py index 60439f6e..52e648de 100644 --- a/uncoder-core/app/translator/core/mixins/rule.py +++ b/uncoder-core/app/translator/core/mixins/rule.py @@ -42,7 +42,7 @@ def parse_mitre_attack(self, tags: list[str]) -> MitreInfoContainer: tag = tag.lower() if tag.startswith("attack."): tag = tag[7::] - if tag.startswith("t"): + if tag[-1].isdigit(): parsed_techniques.append(tag) else: parsed_tactics.append(tag) diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index bb95f9b4..79eef459 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -65,6 +65,8 @@ def __init__( date: Optional[str] = None, output_table_fields: Optional[list[Field]] = None, query_fields: Optional[list[Field]] = None, + function_fields: Optional[list[Field]] = None, + function_fields_map: Optional[dict[str, list[Field]]] = None, license_: Optional[str] = None, severity: Optional[str] = None, references: Optional[list[str]] = None, @@ -76,7 +78,7 @@ def __init__( parsed_logsources: Optional[dict] = None, timeframe: Optional[timedelta] = None, query_period: Optional[timedelta] = None, - mitre_attack: MitreInfoContainer = MitreInfoContainer(), + mitre_attack: Optional[MitreInfoContainer] = None, raw_metainfo_container: Optional[RawMetaInfoContainer] = None, ) -> None: self.id = id_ or str(uuid.uuid4()) @@ -90,11 +92,13 @@ def __init__( self.date = date or datetime.now().date().strftime("%Y-%m-%d") self.output_table_fields = output_table_fields or [] self.query_fields = query_fields or [] + self.function_fields = function_fields or [] + self.function_fields_map = function_fields_map or {} self.license = license_ or "DRL 1.1" self.severity = severity or SeverityType.low self.references = references or [] self.tags = tags or [] - self.mitre_attack = mitre_attack or None + self.mitre_attack = mitre_attack or MitreInfoContainer() self.raw_mitre_attack = raw_mitre_attack or [] self.status = status or "stable" self.false_positives = false_positives or [] @@ -102,7 +106,7 @@ def __init__( self.parsed_logsources = parsed_logsources or {} self.timeframe = timeframe self.query_period = query_period - self.raw_metainfo_container = raw_metainfo_container + self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer() @property def author_str(self) -> str: diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 0ad509d1..2f632b4e 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -24,7 +24,7 @@ from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping -from app.translator.core.models.functions.base import Function +from app.translator.core.models.functions.base import Function, ParsedFunctions from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.models.query_tokens.field import Field @@ -51,6 +51,9 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: raise NotImplementedError("Abstract method") + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + raise NotImplementedError("Abstract method") + class PlatformQueryParser(QueryParser, ABC): mappings: BasePlatformMappings = None @@ -65,16 +68,19 @@ def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]: @staticmethod def get_field_tokens( query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None - ) -> list[Field]: - field_tokens = [] + ) -> tuple[list[Field], list[Field], dict[str, list[Field]]]: + query_field_tokens = [] + function_field_tokens = [] + function_field_tokens_map = {} for token in query_tokens: if isinstance(token, (FieldField, FieldValue, FunctionValue)): - field_tokens.extend(token.fields) + query_field_tokens.extend(token.fields) - if functions: - field_tokens.extend([field for func in functions for field in func.fields]) + for func in functions or []: + function_field_tokens.extend(func.fields) + function_field_tokens_map[func.name] = func.fields - return field_tokens + return query_field_tokens, function_field_tokens, function_field_tokens_map def get_source_mappings( self, field_tokens: list[Field], log_sources: dict[str, list[Union[int, str]]] @@ -85,3 +91,8 @@ def get_source_mappings( ) self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping) return source_mappings + + def get_source_mapping_ids_by_logsources(self, query: str) -> Optional[list[str]]: + _, parsed_logsources, _ = self._parse_query(query=query) + if parsed_logsources: + return self.mappings.get_source_mappings_by_log_sources(parsed_logsources) diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 97709dd0..857c2516 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -428,14 +428,18 @@ def _generate_from_tokenized_query_container_by_source_mapping( self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping ) -> str: unmapped_fields = self.mappings.check_fields_mapping_existence( - query_container.meta_info.query_fields, source_mapping + query_container.meta_info.query_fields, + query_container.meta_info.function_fields_map, + self.platform_functions.manager.supported_render_names, + source_mapping, ) rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix) if source_mapping.raw_log_fields: defined_raw_log_fields = self.generate_raw_log_fields( - fields=query_container.meta_info.query_fields, source_mapping=source_mapping + fields=query_container.meta_info.query_fields + query_container.meta_info.function_fields, + source_mapping=source_mapping, ) prefix += f"\n{defined_raw_log_fields}" query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/default.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/default.yml new file mode 100644 index 00000000..a1db3852 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/default.yml @@ -0,0 +1,2 @@ +platform: CarbonBlack +source: default diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/linux_dns_query.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/linux_dns_query.yml new file mode 100644 index 00000000..e23d35bf --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/linux_dns_query.yml @@ -0,0 +1,8 @@ +platform: CarbonBlack +source: linux_dns_query + + +field_mapping: + User: + - childproc_username + - process_username diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/linux_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/linux_network_connection.yml new file mode 100644 index 00000000..5c6eda13 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/linux_network_connection.yml @@ -0,0 +1,9 @@ +platform: CarbonBlack +source: linux_network_connection + + +field_mapping: + DestinationHostname: + - netconn_domain + - netconn_proxy_domain + DestinationPort: netconn_port diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/macos_dns_query.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/macos_dns_query.yml new file mode 100644 index 00000000..ddff23a5 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/macos_dns_query.yml @@ -0,0 +1,8 @@ +platform: CarbonBlack +source: macos_dns_query + + +field_mapping: + User: + - childproc_username + - process_username diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/macos_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/macos_network_connection.yml new file mode 100644 index 00000000..d61abbf4 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/macos_network_connection.yml @@ -0,0 +1,9 @@ +platform: CarbonBlack +source: macos_network_connection + + +field_mapping: + DestinationHostname: + - netconn_domain + - netconn_proxy_domain + DestinationPort: netconn_port diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_create_remote_thread.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_create_remote_thread.yml new file mode 100644 index 00000000..11a6cf67 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_create_remote_thread.yml @@ -0,0 +1,7 @@ +platform: CarbonBlack +source: windows_create_remote_thread + + +field_mapping: + SourceImage: parent_name + StartModule: modload_name diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_dns_query.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_dns_query.yml new file mode 100644 index 00000000..8f1a84b9 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_dns_query.yml @@ -0,0 +1,8 @@ +platform: CarbonBlack +source: windows_dns_query + + +field_mapping: + User: + - childproc_username + - process_username diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_file_event.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_file_event.yml new file mode 100644 index 00000000..86fcf3a5 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_file_event.yml @@ -0,0 +1,8 @@ +platform: CarbonBlack +source: windows_file_event + + +field_mapping: + User: + - childproc_username + - process_username diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_image_load.yml new file mode 100644 index 00000000..11199a15 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_image_load.yml @@ -0,0 +1,6 @@ +platform: CarbonBlack +source: windows_image_load + + +field_mapping: + OriginalFileName: process_original_filename diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_network_connection.yml new file mode 100644 index 00000000..8017db4f --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_network_connection.yml @@ -0,0 +1,9 @@ +platform: CarbonBlack +source: windows_network_connection + + +field_mapping: + DestinationHostname: + - netconn_domain + - netconn_proxy_domain + DestinationPort: netconn_port diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_process_creation.yml new file mode 100644 index 00000000..cb4fc2c8 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_process_creation.yml @@ -0,0 +1,14 @@ +platform: CarbonBlack +source: windows_process_creation + + +field_mapping: + Hashes: + - md5 + - filewrite_md5 + - childproc_md5 + - parent_md5 + User: + - childproc_username + - process_username + OriginalFileName: process_original_filename \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_registry_event.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_registry_event.yml new file mode 100644 index 00000000..ff1b0aee --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_registry_event.yml @@ -0,0 +1,9 @@ +platform: CarbonBlack +source: windows_registry_event + + +field_mapping: + TargetObject: regmod_name + User: + - childproc_username + - process_username diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_security.yml new file mode 100644 index 00000000..6e288c0b --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_security.yml @@ -0,0 +1,17 @@ +platform: CarbonBlack +source: windows_security + + +field_mapping: + AccountName: + - process_username + - childproc_username + ComputerName: device_name + NewProcessName: process_name + DeviceDescription: + - process_product_name + - process_product_version + - process_publisher + - process_file_description + DestPort: netconn_port + UserID: parent_name \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_sysmon.yml new file mode 100644 index 00000000..65778b83 --- /dev/null +++ b/uncoder-core/app/translator/mappings/platforms/carbonblack/windows_sysmon.yml @@ -0,0 +1,51 @@ +platform: CarbonBlack +source: windows_sysmon + + + +field_mapping: + CommandLine: process_cmdline + Image: process_name + ParentImage: parent_name + Company: process_publisher + Description: + - process_product_name + - process_product_version + - process_publisher + - process_file_description + DestinationHostname: + - netconn_domain + - netconn_proxy_domain + DestinationIp: + - netconn_ipv4 + - netconn_ipv6 + DestinationIsIpv6: ipaddr + Hashes: + - md5 + - filewrite_md5 + - childproc_md5 + - parent_md5 + IntegrityLevel: process_integrity_level + ParentCommandLine: parent_cmdline + Product: + - process_product_name + - process_file_description + SourceIp: + - netconn_ipv4 + - netconn_ipv6 + - netconn_local_ipv4 + - netconn_local_ipv6 + SourcePort: netconn_port + TargetFilename: filemod_name + User: childproc_username;process_username + OriginalFileName: process_original_filename + Signature: + - childproc_publisher + - filemod_publisher + - modload_publisher + - parent_publisher + - process_publisher + ImageLoaded: modload_name + StartModule: modload_name + TargetImage: filemod_name + FileVersion: process_product_version \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_bits_client.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_bits_client.yml index 2baca60b..5ea62c7e 100644 --- a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_bits_client.yml +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_bits_client.yml @@ -1,6 +1,8 @@ platform: ElasticSearch source: windows_bits_client +conditions: + winlog.channel: 'Microsoft-Windows-Bits-Client/Operational' log_source: index: [winlogbeat-*, logs-*] diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_image_load.yml index 265cc0ac..38e615c1 100644 --- a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_image_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_image_load.yml @@ -1,6 +1,10 @@ platform: ElasticSearch source: windows_image_load +conditions: + event.action: + - 'Image loaded (rule: ImageLoad)' + - 'load' log_source: index: [winlogbeat-*, logs-*] diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_ldap_debug.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_ldap_debug.yml index fb3e7175..1a8d1b6b 100644 --- a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_ldap_debug.yml +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_ldap_debug.yml @@ -1,6 +1,8 @@ platform: ElasticSearch source: windows_ldap_debug +conditions: + winlog.channel: 'Microsoft-Windows-LDAP-Client/Debug' log_source: index: [winlogbeat-*, logs-*] diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_network_connection.yml index 93953c24..dcac9463 100644 --- a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_network_connection.yml +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_network_connection.yml @@ -1,6 +1,8 @@ platform: ElasticSearch source: windows_network_connection +conditions: + event.category: 'network' log_source: index: [winlogbeat-*, logs-*] diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_ntlm.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_ntlm.yml index 19ff651d..d8de2ac2 100644 --- a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_ntlm.yml +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_ntlm.yml @@ -1,6 +1,8 @@ platform: ElasticSearch source: windows_ntlm +conditions: + winlog.provider_name: 'Microsoft-Windows-NTLM/Operational' log_source: index: [winlogbeat-*, logs-*] diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_process_creation.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_process_creation.yml index 31e90d94..c0c1dea8 100644 --- a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_process_creation.yml +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_process_creation.yml @@ -1,6 +1,8 @@ platform: ElasticSearch source: windows_process_creation +conditions: + event.category: 'process' log_source: index: [winlogbeat-*, logs-*] diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_security.yml index 6105605f..5a01016a 100644 --- a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_security.yml +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_security.yml @@ -1,6 +1,8 @@ platform: ElasticSearch source: windows_security +conditions: + winlog.channel : "Security" log_source: index: [winlogbeat-*, logs-*] diff --git a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_sysmon.yml index 81f9df80..edb7e997 100644 --- a/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_sysmon.yml +++ b/uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_sysmon.yml @@ -1,6 +1,8 @@ platform: ElasticSearch source: windows_sysmon +conditions: + winlog.channel: 'Microsoft-Windows-Sysmon/Operational' log_source: index: [winlogbeat-*, logs-*] diff --git a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py index 5b3a7041..44800cf9 100644 --- a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py @@ -17,7 +17,7 @@ """ import re -from typing import Union +from typing import Optional, Union from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.models.functions.base import ParsedFunctions @@ -105,8 +105,8 @@ def __parse_log_sources(self, query: str) -> tuple[dict[str, Union[list[str], li return log_sources, query - def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[int]]], ParsedFunctions]: - query = self.__clean_query(text) + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + query = self.__clean_query(query) self.__check_table(query) query, functions = self.platform_functions.parse(query) log_sources, query = self.__parse_log_sources(query) @@ -115,9 +115,13 @@ def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources, functions = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) - field_tokens = self.get_field_tokens(query_tokens, functions.functions) - source_mappings = self.get_source_mappings(field_tokens, log_sources) + query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( + query_tokens, functions.functions + ) + source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = field_tokens + meta_info.query_fields = query_field_tokens + meta_info.function_fields = function_field_tokens + meta_info.function_fields_map = function_field_tokens_map meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py index 5fb57284..77ef79f4 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py @@ -17,7 +17,9 @@ """ import re +from typing import Optional, Union +from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.lucene.tokenizer import LuceneTokenizer @@ -31,7 +33,7 @@ class LuceneQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: log_sources = {} for source_type in self.log_source_key_types: pattern = self.log_source_pattern.replace("___source_type___", source_type) @@ -43,14 +45,14 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: pos_end = search.end() query = query[:pos_start] + query[pos_end:] - return query, log_sources + return query, log_sources, None def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - query, log_sources = self._parse_query(raw_query_container.query) + query, log_sources, _ = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) - field_tokens = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(field_tokens, log_sources) + query_field_tokens, _, _ = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(query_field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = field_tokens + meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py b/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py index b56f5bee..8be19ffe 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py +++ b/uncoder-core/app/translator/platforms/base/lucene/tokenizer.py @@ -38,6 +38,7 @@ class LuceneTokenizer(QueryTokenizer, ANDLogicOperatorMixin): ":>": OperatorType.GT, ":<": OperatorType.LT, ":": OperatorType.EQ, + "==": OperatorType.EQ, } multi_value_operators_map: ClassVar[dict[str, str]] = {":": OperatorType.EQ} @@ -61,7 +62,7 @@ class LuceneTokenizer(QueryTokenizer, ANDLogicOperatorMixin): multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>[:a-zA-Z\"\*0-9=+%#№;\-_\/\\'\,.$&^@!\(\[\]\s|]+)\)""" multi_value_check_pattern = r"___field___\s*___operator___\s*\(" - multi_value_delimiter_pattern = r"\s+OR\s+" + multi_value_delimiter_pattern = r"\s+(?:OR|or)\s+" escape_manager = lucene_escape_manager @@ -77,7 +78,9 @@ def create_field_value(field_name: str, operator: Identifier, value: Union[str, @staticmethod def clean_multi_value(value: str) -> str: - return value.strip('"') if value.startswith('"') and value.endswith('"') else value + value = value.replace("\n", "").replace(" ", "") + value = value.strip('"') if value.startswith('"') and value.endswith('"') else value + return value.strip() def get_operator_and_value( # noqa: PLR0911 self, match: re.Match, mapped_operator: str = OperatorType.EQ, operator: Optional[str] = None diff --git a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py index 27a1559d..f56af913 100644 --- a/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/parsers/spl.py @@ -29,6 +29,7 @@ class SplQueryParser(PlatformQueryParser): log_source_pattern = r"^___source_type___\s*=\s*(?:\"(?P[%a-zA-Z_*:0-9\-/]+)\"|(?P[%a-zA-Z_*:0-9\-/]+))(?:\s+(?:and|or)\s+|\s+)?" # noqa: E501 + rule_name_pattern = r"`(?P(?:[:a-zA-Z*0-9=+%#\-_/,;`?~‘\'.<>$&^@!\]\[()\s])*)`" # noqa: RUF001 log_source_key_types = ("index", "source", "sourcetype", "sourcecategory") platform_functions: SplFunctions = None @@ -53,6 +54,9 @@ def _parse_log_sources(self, query: str) -> tuple[dict[str, list[str]], str]: return log_sources, query def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFunctions]: + if re.match(self.rule_name_pattern, query): + search = re.search(self.rule_name_pattern, query, flags=re.IGNORECASE) + query = query[: search.start()] + query[search.end() :] query = query.strip() log_sources, query = self._parse_log_sources(query) query, functions = self.platform_functions.parse(query) @@ -68,9 +72,13 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain query, log_sources, functions = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) - field_tokens = self.get_field_tokens(query_tokens, functions.functions) - source_mappings = self.get_source_mappings(field_tokens, log_sources) + query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( + query_tokens, functions.functions + ) + source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = field_tokens + meta_info.query_fields = query_field_tokens + meta_info.function_fields = function_field_tokens + meta_info.function_fields_map = function_field_tokens_map meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py index 735f95c6..2b5854cb 100644 --- a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py @@ -17,7 +17,9 @@ """ import re +from typing import Optional, Union +from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.sql.tokenizer import SqlTokenizer @@ -30,22 +32,22 @@ class SqlQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*--.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: log_source = {"table": []} if re.search(self.query_delimiter_pattern, query, flags=re.IGNORECASE): table_search = re.search(self.table_pattern, query) table = table_search.group("table") log_source["table"] = [table] - return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source + return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source, None - return query, log_source + return query, log_source, None def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - query, log_sources = self._parse_query(raw_query_container.query) + query, log_sources, _ = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) - field_tokens = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(field_tokens, log_sources) + query_field_tokens, _, _ = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(query_field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = field_tokens + meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/carbonblack/__init__.py b/uncoder-core/app/translator/platforms/carbonblack/__init__.py index 715f3a24..ebc8a99c 100644 --- a/uncoder-core/app/translator/platforms/carbonblack/__init__.py +++ b/uncoder-core/app/translator/platforms/carbonblack/__init__.py @@ -1 +1,2 @@ +from app.translator.platforms.carbonblack.renders.carbonblack import CarbonBlackQueryRender # noqa: F401 from app.translator.platforms.carbonblack.renders.carbonblack_cti import CarbonBlackCTI # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/carbonblack/const.py b/uncoder-core/app/translator/platforms/carbonblack/const.py index 8f1d8958..99175b65 100644 --- a/uncoder-core/app/translator/platforms/carbonblack/const.py +++ b/uncoder-core/app/translator/platforms/carbonblack/const.py @@ -1,3 +1,5 @@ +from app.translator.core.models.platform_details import PlatformDetails + CARBON_BLACK_QUERY_DETAILS = { "platform_id": "carbonblack", "name": "Carbon Black Cloud", @@ -5,3 +7,5 @@ "group_id": "carbonblack-pack", "platform_name": "Query (Cloud)", } + +carbonblack_query_details = PlatformDetails(**CARBON_BLACK_QUERY_DETAILS) diff --git a/uncoder-core/app/translator/platforms/carbonblack/escape_manager.py b/uncoder-core/app/translator/platforms/carbonblack/escape_manager.py new file mode 100644 index 00000000..5fd8662c --- /dev/null +++ b/uncoder-core/app/translator/platforms/carbonblack/escape_manager.py @@ -0,0 +1,20 @@ +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.escape_manager import EscapeManager +from app.translator.core.models.escape_details import EscapeDetails + + +class CarbonBlackEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [ + EscapeDetails( + pattern='([\s+\\-=&?!|(){}.\\[\\]^"~:/]|(?", + ) + ], + ValueType.regex_value: [EscapeDetails(pattern=r"([$^*+()\[\]{}|.?\-\\])", escape_symbols=r"\\\1")], + } + + +carbon_black_escape_manager = CarbonBlackEscapeManager() diff --git a/uncoder-core/app/translator/platforms/carbonblack/mapping.py b/uncoder-core/app/translator/platforms/carbonblack/mapping.py new file mode 100644 index 00000000..b31384b9 --- /dev/null +++ b/uncoder-core/app/translator/platforms/carbonblack/mapping.py @@ -0,0 +1,18 @@ +from app.translator.core.mapping import BaseStrictLogSourcesPlatformMappings, LogSourceSignature +from app.translator.platforms.carbonblack.const import carbonblack_query_details + + +class CarbonBlackLogSourceSignature(LogSourceSignature): + def is_suitable(self) -> bool: + return True + + def __str__(self) -> str: + return "" + + +class CarbonBlackMappings(BaseStrictLogSourcesPlatformMappings): + def prepare_log_source_signature(self, mapping: dict) -> CarbonBlackLogSourceSignature: + ... + + +carbonblack_query_mappings = CarbonBlackMappings(platform_dir="carbonblack", platform_details=carbonblack_query_details) diff --git a/uncoder-core/app/translator/platforms/carbonblack/renders/carbonblack.py b/uncoder-core/app/translator/platforms/carbonblack/renders/carbonblack.py new file mode 100644 index 00000000..df366c3e --- /dev/null +++ b/uncoder-core/app/translator/platforms/carbonblack/renders/carbonblack.py @@ -0,0 +1,103 @@ +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.custom_types.values import ValueType +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.managers import render_manager +from app.translator.platforms.carbonblack.const import carbonblack_query_details +from app.translator.platforms.carbonblack.mapping import CarbonBlackMappings, carbonblack_query_mappings +from app.translator.platforms.carbonblack.str_value_manager import ( + CarbonBlackStrValueManager, + carbon_black_str_value_manager, +) + + +class CarbonBlackFieldValueRender(BaseFieldValueRender): + details: PlatformDetails = carbonblack_query_details + str_value_manager: CarbonBlackStrValueManager = carbon_black_str_value_manager + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f'"{value}"' + + @staticmethod + def _wrap_int_value(value: int) -> str: + return f'"{value}"' + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.equal_modifier(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field}:{value}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = [ + self._pre_process_value(field, val, value_type=ValueType.value, wrap_str=True, wrap_int=True) + for val in value + ] + return f"(NOT {field}:({self.or_token.join(values)})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"(NOT {field}:{self.apply_value(value)})" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.or_token.join( + [f"*{self._pre_process_value(field, val, value_type=ValueType.value)}*" for val in value] + ) + return f"{field}:({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value) + return f"{field}:*{value}*" + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.or_token.join( + [f"*{self._pre_process_value(field, val, value_type=ValueType.value)}" for val in value] + ) + return f"{field}:({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value) + return f"{field}:*{value}" + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.or_token.join( + [f"{self._pre_process_value(field, val, value_type=ValueType.value)}*" for val in value] + ) + return f"{field}:({values}" + value = self._pre_process_value(field, value, value_type=ValueType.value) + return f"{field}:{value}*" + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.regex_value) + return f"{field}:/{value}/" + + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.keywords(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.value) + return f"(*{value}*)" + + def is_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.is_none(field=field, value=v) for v in value)})" + return f"NOT _exists_:{field}" + + def is_not_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.is_not_none(field=field, value=v) for v in value)})" + return f"_exists_:{field}" + + +@render_manager.register +class CarbonBlackQueryRender(PlatformQueryRender): + details: PlatformDetails = carbonblack_query_details + mappings: CarbonBlackMappings = carbonblack_query_mappings + + or_token = "OR" + and_token = "AND" + not_token = "NOT" + + comment_symbol = "//" + + field_value_render = CarbonBlackFieldValueRender(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/carbonblack/renders/carbonblack_cti.py b/uncoder-core/app/translator/platforms/carbonblack/renders/carbonblack_cti.py index 489a1288..225434ab 100644 --- a/uncoder-core/app/translator/platforms/carbonblack/renders/carbonblack_cti.py +++ b/uncoder-core/app/translator/platforms/carbonblack/renders/carbonblack_cti.py @@ -20,13 +20,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render_cti import RenderCTI from app.translator.managers import render_cti_manager -from app.translator.platforms.carbonblack.const import CARBON_BLACK_QUERY_DETAILS +from app.translator.platforms.carbonblack.const import carbonblack_query_details from app.translator.platforms.carbonblack.mappings.carbonblack_cti import DEFAULT_CARBONBLACK_MAPPING @render_cti_manager.register class CarbonBlackCTI(RenderCTI): - details: PlatformDetails = PlatformDetails(**CARBON_BLACK_QUERY_DETAILS) + details: PlatformDetails = carbonblack_query_details field_value_template: str = "{key}:{value}" or_operator: str = " OR " diff --git a/uncoder-core/app/translator/platforms/carbonblack/str_value_manager.py b/uncoder-core/app/translator/platforms/carbonblack/str_value_manager.py new file mode 100644 index 00000000..0f675093 --- /dev/null +++ b/uncoder-core/app/translator/platforms/carbonblack/str_value_manager.py @@ -0,0 +1,38 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" +from typing import ClassVar + +from app.translator.core.str_value_manager import ( + BaseSpecSymbol, + SingleSymbolWildCard, + StrValueManager, + UnboundLenWildCard, +) +from app.translator.platforms.carbonblack.escape_manager import CarbonBlackEscapeManager, carbon_black_escape_manager + + +class CarbonBlackStrValueManager(StrValueManager): + escape_manager: CarbonBlackEscapeManager = carbon_black_escape_manager + str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { + "?": SingleSymbolWildCard, + "*": UnboundLenWildCard, + } + + +carbon_black_str_value_manager = CarbonBlackStrValueManager() diff --git a/uncoder-core/app/translator/platforms/chronicle/const.py b/uncoder-core/app/translator/platforms/chronicle/const.py index d788860a..142eaae7 100644 --- a/uncoder-core/app/translator/platforms/chronicle/const.py +++ b/uncoder-core/app/translator/platforms/chronicle/const.py @@ -20,18 +20,18 @@ $e }""" -PLATFORM_DETAILS = {"group_id": "chronicle-pack", "group_name": "Chronicle Security", "alt_platform_name": "UDM"} +PLATFORM_DETAILS = {"group_id": "chronicle-pack", "group_name": "Google SecOps", "alt_platform_name": "UDM"} CHRONICLE_QUERY_DETAILS = { "platform_id": "chronicle-yaral-query", - "name": "Chronicle Security Query", + "name": "Google SecOps Query", "platform_name": "Query (UDM)", **PLATFORM_DETAILS, } CHRONICLE_RULE_DETAILS = { "platform_id": "chronicle-yaral-rule", - "name": "Chronicle Security Rule", + "name": "Google SecOps Rule", "platform_name": "Rule (YARA-L)", "first_choice": 0, **PLATFORM_DETAILS, diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py index 7c50cb06..0cc1af82 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py @@ -35,9 +35,9 @@ class ChronicleQueryParser(PlatformQueryParser): def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query_tokens = self.get_query_tokens(raw_query_container.query) - field_tokens = self.get_field_tokens(query_tokens) - source_mappings = self.get_source_mappings(field_tokens, {}) + query_field_tokens, _, _ = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(query_field_tokens, {}) meta_info = raw_query_container.meta_info - meta_info.query_fields = field_tokens + meta_info.query_fields = query_field_tokens meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py index 0d03c747..1c923aaf 100644 --- a/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py @@ -19,6 +19,7 @@ import re from app.translator.core.exceptions.parser import TokenizerGeneralException +from app.translator.core.mitre import MitreConfig, MitreInfoContainer from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.managers import parser_manager @@ -37,6 +38,7 @@ class ChronicleRuleParser(ChronicleQueryParser): event_name_pattern = r"condition:\n\s*(?P\$[a-zA-Z_0-9]+)\n" mappings: ChronicleMappings = chronicle_rule_mappings tokenizer = ChronicleRuleTokenizer() + mitre_config: MitreConfig = MitreConfig() def __parse_rule(self, rule: str) -> tuple[str, str, str]: if (rule_name_search := re.search(self.rule_name_pattern, rule)) is None: @@ -63,28 +65,52 @@ def __prepare_title(name: str) -> str: return " ".join(name.split("_")).title() @staticmethod - def __parse_meta_info(meta_info_str: str) -> tuple[str, list[str], list[str]]: - references = tags = [] - description = None + def __parse_meta_info(meta_info_str: str) -> dict: + parsed_meta_info = {} + for info in meta_info_str.strip(" ").strip("\n").split("\n"): key, value = info.split(" = ") key = key.strip(" ") - if key == "description": - description = value.strip(" ") + if key in ("description", "license", "version", "sigma_id", "status", "severity", "created"): + parsed_meta_info[key] = value.strip(" ").strip('"') elif key == "reference": - references = [value.strip(" ").strip('"')] - elif key == "tags": - tags = [i.strip(" ").strip('"') for i in value.split(",")] - - return description, references, tags + parsed_meta_info[key] = [value.strip(" ").strip('"')] + elif key in ("tags", "author"): + parsed_meta_info[key] = [i.strip(" ").strip('"') for i in value.split(",")] + + return parsed_meta_info + + def parse_mitre_attack_from_tags(self, tags: list) -> MitreInfoContainer: + parsed_techniques = [] + parsed_tactics = [] + for tag in set(tags): + tag = tag.lower() + if tag.startswith("attack."): + tag = tag[7::] + if tag[-1].isdigit(): + parsed_techniques.append(tag) + else: + parsed_tactics.append(tag) + return self.mitre_config.get_mitre_info(tactics=parsed_tactics, techniques=parsed_techniques) def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: query, rule_name, meta_info_str = self.__parse_rule(text) - description, references, tags = self.__parse_meta_info(meta_info_str) + parsed_meta_info = self.__parse_meta_info(meta_info_str) + return RawQueryContainer( query=query, language=language, meta_info=MetaInfoContainer( - title=self.__prepare_title(rule_name), description=description, references=references, tags=tags + id_=parsed_meta_info.get("sigma_id"), + title=self.__prepare_title(rule_name), + description=parsed_meta_info.get("description"), + author=parsed_meta_info.get("author"), + date=parsed_meta_info.get("created"), + license_=parsed_meta_info.get("license"), + severity=parsed_meta_info.get("severity"), + references=parsed_meta_info.get("reference"), + tags=parsed_meta_info.get("tags"), + status=parsed_meta_info.get("status"), + mitre_attack=self.parse_mitre_attack_from_tags(parsed_meta_info.get("tags") or []), ), ) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py index 91a7d362..f13f11f3 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/__init__.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/__init__.py @@ -3,10 +3,12 @@ ElasticSearchRuleTOMLParser, # noqa: F401 ) from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser # noqa: F401 +from app.translator.platforms.elasticsearch.parsers.elasticsearch_eql import ElasticSearchEQLQueryParser # noqa: F401 from app.translator.platforms.elasticsearch.renders.detection_rule import ElasticSearchRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.elasticsearch_cti import ElasticsearchCTI # noqa: F401 +from app.translator.platforms.elasticsearch.renders.elasticsearch_eql import ElasticSearchEQLQueryRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.esql import ESQLQueryRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.esql_rule import ESQLRuleRender # noqa: F401 from app.translator.platforms.elasticsearch.renders.kibana import KibanaRuleRender # noqa: F401 diff --git a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py index 2109ed2e..993fdcfa 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/escape_manager.py @@ -20,4 +20,12 @@ class ESQLQueryEscapeManager(EscapeManager): } +class EQLQueryEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\")], + ValueType.regex_value: [EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\")], + } + + esql_query_escape_manager = ESQLQueryEscapeManager() +eql_query_escape_manager = EQLQueryEscapeManager() diff --git a/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py new file mode 100644 index 00000000..377b1e08 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/parsers/elasticsearch_eql.py @@ -0,0 +1,37 @@ +import re + +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser +from app.translator.managers import parser_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings +from app.translator.platforms.elasticsearch.const import elastic_eql_query_details +from app.translator.platforms.elasticsearch.mapping import elastic_eql_query_mappings +from app.translator.platforms.elasticsearch.tokenizer import ElasticSearchEQLTokenizer + + +@parser_manager.register_supported_by_roota +class ElasticSearchEQLQueryParser(PlatformQueryParser): + details: PlatformDetails = elastic_eql_query_details + tokenizer = ElasticSearchEQLTokenizer() + mappings: LuceneMappings = elastic_eql_query_mappings + query_delimiter_pattern = r"\swhere\s" + + def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: + log_source = {"category": []} + if re.search(self.query_delimiter_pattern, query, flags=re.IGNORECASE): + sp_query = re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE) + if sp_query[0].lower() != "all": + log_source["category"].append(sp_query[0]) + return sp_query[1], log_source + return query, log_source + + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + query, log_sources = self._parse_query(raw_query_container.query) + query_tokens = self.get_query_tokens(query) + query_field_tokens, _, _ = self.get_field_tokens(query_tokens) + source_mappings = self.get_source_mappings(query_field_tokens, log_sources) + meta_info = raw_query_container.meta_info + meta_info.query_fields = query_field_tokens + meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] + return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py new file mode 100644 index 00000000..0b3f8f34 --- /dev/null +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch_eql.py @@ -0,0 +1,174 @@ +from typing import Optional, Union + +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.const import QUERY_TOKEN_TYPE +from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType +from app.translator.core.custom_types.values import ValueType +from app.translator.core.mapping import LogSourceSignature, SourceMapping +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.core.models.query_tokens.field_value import FieldValue +from app.translator.core.models.query_tokens.identifier import Identifier +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.core.str_value_manager import StrValueManager +from app.translator.managers import render_manager +from app.translator.platforms.base.lucene.mapping import LuceneMappings +from app.translator.platforms.elasticsearch.const import elastic_eql_query_details +from app.translator.platforms.elasticsearch.mapping import elastic_eql_query_mappings +from app.translator.platforms.elasticsearch.str_value_manager import eql_str_value_manager + + +class ElasticSearchEQLFieldValue(BaseFieldValueRender): + details: PlatformDetails = elastic_eql_query_details + str_value_manager: StrValueManager = eql_str_value_manager + list_token = ", " + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f'"{value}"' + + @staticmethod + def _wrap_int_value(value: int) -> str: + return f'"{value}"' + + def apply_field(self, field: str) -> str: + if field.count("-") > 0 or field.count(" ") > 0 or field[0].isdigit(): + return f"`{field}`" + if field.endswith(".text"): + return field[:-5] + return field + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.list_token.join( + self._pre_process_value(field, v, value_type=ValueType.value, wrap_str=True, wrap_int=True) + for v in value + ) + return f"{self.apply_field(field)} : ({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{self.apply_field(field)} : {value}" + + def less_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{self.apply_field(field)} < {value}" + + def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{self.apply_field(field)} <= {value}" + + def greater_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{self.apply_field(field)} > {value}" + + def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{self.apply_field(field)} >= {value}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.list_token.join( + self._pre_process_value(field, v, value_type=ValueType.value, wrap_str=True, wrap_int=True) + for v in value + ) + return f"{self.apply_field(field)} != ({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{self.apply_field(field)} != {value}" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.list_token.join( + f'"*{self._pre_process_value(field, v, value_type=ValueType.value)}*"' for v in value + ) + return f"{self.apply_field(field)} : ({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value) + return f'{self.apply_field(field)} : "*{value}*"' + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.list_token.join( + f'"*{self._pre_process_value(field, v, value_type=ValueType.value)}"' for v in value + ) + return f"{self.apply_field(field)} : ({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value) + return f'{self.apply_field(field)} : "*{value}"' + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.list_token.join( + f'"{self._pre_process_value(field, v, value_type=ValueType.value)}*"' for v in value + ) + return f"{self.apply_field(field)} : ({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value) + return f'{self.apply_field(field)} : "{value}*"' + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})" + value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_int=True) + return f'{self.apply_field(field)} regex~ "{value}[^z].?"' + + def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + return f"({self.or_token.join(self.keywords(field=field, value=v) for v in value)})" + return self._pre_process_value(field, value, wrap_str=True) + + def is_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 + return f"{self.apply_field(field)} == null" + + def is_not_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 + return f"{self.apply_field(field)} != null" + + +@render_manager.register +class ElasticSearchEQLQueryRender(PlatformQueryRender): + details: PlatformDetails = elastic_eql_query_details + mappings: LuceneMappings = elastic_eql_query_mappings + or_token = "or" + and_token = "and" + not_token = "not" + comment_symbol = "//" + field_value_render = ElasticSearchEQLFieldValue(or_token=or_token) + + def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002 + return "any where " + + def in_brackets(self, raw_list: list[QUERY_TOKEN_TYPE]) -> list[QUERY_TOKEN_TYPE]: + return [Identifier(token_type=GroupType.L_PAREN), *raw_list, Identifier(token_type=GroupType.R_PAREN)] + + def _generate_from_tokenized_query_container_by_source_mapping( + self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping + ) -> str: + unmapped_fields = self.mappings.check_fields_mapping_existence( + query_container.meta_info.query_fields, + query_container.meta_info.function_fields_map, + self.platform_functions.manager.supported_render_names, + source_mapping, + ) + rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping) + prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix) + + if source_mapping.raw_log_fields: + defined_raw_log_fields = self.generate_raw_log_fields( + fields=query_container.meta_info.query_fields + query_container.meta_info.function_fields, + source_mapping=source_mapping, + ) + prefix += f"\n{defined_raw_log_fields}" + if source_mapping.conditions: + for field, value in source_mapping.conditions.items(): + tokens = self.in_brackets(query_container.tokens) + extra_tokens = [ + FieldValue(source_name=field, operator=Identifier(token_type=OperatorType.EQ), value=value), + Identifier(token_type=LogicalOperatorType.AND), + ] + query_container.tokens = self.in_brackets([*extra_tokens, *tokens]) + query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) + not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported + return self.finalize_query( + prefix=prefix, + query=query, + functions=rendered_functions.rendered, + not_supported_functions=not_supported_functions, + unmapped_fields=unmapped_fields, + meta_info=query_container.meta_info, + source_mapping=source_mapping, + ) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py index e1b8708a..ca38d5d7 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/str_value_manager.py @@ -23,12 +23,20 @@ ReDigitalSymbol, ReWhiteSpaceSymbol, ReWordSymbol, + SingleSymbolWildCard, + StrValue, StrValueManager, + UnboundLenWildCard, +) +from app.translator.platforms.elasticsearch.escape_manager import ( + EQLQueryEscapeManager, + ESQLQueryEscapeManager, + eql_query_escape_manager, + esql_query_escape_manager, ) -from app.translator.platforms.elasticsearch.escape_manager import ESQLQueryEscapeManager, esql_query_escape_manager -class ESQLQueryStrValueManager(StrValueManager): +class ESQLStrValueManager(StrValueManager): escape_manager: ESQLQueryEscapeManager = esql_query_escape_manager re_str_alpha_num_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { "w": ReWordSymbol, @@ -37,4 +45,17 @@ class ESQLQueryStrValueManager(StrValueManager): } -esql_query_str_value_manager = ESQLQueryStrValueManager() +class EQLStrValueManager(StrValueManager): + escape_manager: EQLQueryEscapeManager = eql_query_escape_manager + str_spec_symbols_map: ClassVar[dict[str, type[BaseSpecSymbol]]] = { + "?": SingleSymbolWildCard, + "*": UnboundLenWildCard, + } + + def from_str_to_container(self, value: str) -> StrValue: + split = [self.str_spec_symbols_map[char]() if char in self.str_spec_symbols_map else char for char in value] + return StrValue(value, self._concat(split)) + + +esql_str_value_manager = ESQLStrValueManager() +eql_str_value_manager = EQLStrValueManager() diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index 138e56c6..f9b3e942 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -232,7 +232,10 @@ def _generate_from_tokenized_query_container_by_source_mapping( self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping ) -> str: unmapped_fields = self.mappings.check_fields_mapping_existence( - query_container.meta_info.query_fields, source_mapping + query_container.meta_info.query_fields, + query_container.meta_info.function_fields_map, + self.platform_functions.manager.supported_render_names, + source_mapping, ) is_event_type_set = False field_values = [token for token in query_container.tokens if isinstance(token, FieldValue)] diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index b81f5453..c9172b58 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -244,7 +244,10 @@ def _generate_from_tokenized_query_container_by_source_mapping( self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping ) -> str: unmapped_fields = self.mappings.check_fields_mapping_existence( - query_container.meta_info.query_fields, source_mapping + query_container.meta_info.query_fields, + query_container.meta_info.function_fields_map, + self.platform_functions.manager.supported_render_names, + source_mapping, ) prefix = self.generate_prefix(source_mapping.log_source_signature) if "product" in query_container.meta_info.parsed_logsources: diff --git a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py index 4f6fb9d9..ddf2fcd1 100644 --- a/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/parsers/logscale.py @@ -43,9 +43,13 @@ def _parse_query(self, query: str) -> tuple[str, ParsedFunctions]: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, functions = self._parse_query(query=raw_query_container.query) query_tokens = self.get_query_tokens(query) - field_tokens = self.get_field_tokens(query_tokens, functions.functions) - source_mappings = self.get_source_mappings(field_tokens, {}) + query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( + query_tokens, functions.functions + ) + source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, {}) meta_info = raw_query_container.meta_info - meta_info.query_fields = field_tokens + meta_info.query_fields = query_field_tokens + meta_info.function_fields = function_field_tokens + meta_info.function_fields_map = function_field_tokens_map meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index 24d522e9..ecebd04b 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -16,6 +16,8 @@ ----------------------------------------------------------------- """ +from typing import Optional, Union + from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer @@ -36,7 +38,7 @@ class MicrosoftSentinelQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFunctions]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: table, query, functions = self.platform_functions.parse(query) log_sources = {"table": [table]} return query, log_sources, functions @@ -44,9 +46,13 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFun def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: query, log_sources, functions = self._parse_query(query=raw_query_container.query) query_tokens = self.get_query_tokens(query) - field_tokens = self.get_field_tokens(query_tokens, functions.functions) - source_mappings = self.get_source_mappings(field_tokens, log_sources) + query_field_tokens, function_field_tokens, function_field_tokens_map = self.get_field_tokens( + query_tokens, functions.functions + ) + source_mappings = self.get_source_mappings(query_field_tokens + function_field_tokens, log_sources) meta_info = raw_query_container.meta_info - meta_info.query_fields = field_tokens + meta_info.query_fields = query_field_tokens + meta_info.function_fields = function_field_tokens + meta_info.function_fields_map = function_field_tokens_map meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions) From d87ec7621602ea662c0ff2d5f7ae95105625b4c3 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:03:02 +0300 Subject: [PATCH 4/9] gis-8825 added sentinel one power query render --- .../platforms/sentinel_one/__init__.py | 3 + .../platforms/sentinel_one/const.py | 1 - .../platforms/sentinel_one/escape_manager.py | 15 +++ .../platforms/sentinel_one/mapping.py | 20 ++++ .../renders/sentinel_one_power_query.py | 110 ++++++++++++++++++ .../sentinel_one/str_value_manager.py | 31 +++++ 6 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 uncoder-core/app/translator/platforms/sentinel_one/escape_manager.py create mode 100644 uncoder-core/app/translator/platforms/sentinel_one/mapping.py create mode 100644 uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py create mode 100644 uncoder-core/app/translator/platforms/sentinel_one/str_value_manager.py diff --git a/uncoder-core/app/translator/platforms/sentinel_one/__init__.py b/uncoder-core/app/translator/platforms/sentinel_one/__init__.py index 0ba5cbed..d73e2978 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/__init__.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/__init__.py @@ -1 +1,4 @@ from app.translator.platforms.sentinel_one.renders.s1_cti import S1EventsCTI # noqa: F401 +from app.translator.platforms.sentinel_one.renders.sentinel_one_power_query import ( + SentinelOnePowerQueryRender, # noqa: F401 +) diff --git a/uncoder-core/app/translator/platforms/sentinel_one/const.py b/uncoder-core/app/translator/platforms/sentinel_one/const.py index 4e1b6d65..869aff36 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/const.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/const.py @@ -1,6 +1,5 @@ from app.translator.core.models.platform_details import PlatformDetails - PLATFORM_DETAILS = {"group_id": "sentinel-one", "group_name": "SentinelOne"} SENTINEL_ONE_EVENTS_QUERY_DETAILS = { diff --git a/uncoder-core/app/translator/platforms/sentinel_one/escape_manager.py b/uncoder-core/app/translator/platforms/sentinel_one/escape_manager.py new file mode 100644 index 00000000..45232ef1 --- /dev/null +++ b/uncoder-core/app/translator/platforms/sentinel_one/escape_manager.py @@ -0,0 +1,15 @@ +from typing import ClassVar + +from app.translator.core.custom_types.values import ValueType +from app.translator.core.escape_manager import EscapeManager +from app.translator.core.models.escape_details import EscapeDetails + + +class SentinelOnePowerQueryEscapeManager(EscapeManager): + escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { + ValueType.value: [EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\")], + ValueType.regex_value: [EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\")], + } + + +sentinel_one_power_query_escape_manager = SentinelOnePowerQueryEscapeManager() diff --git a/uncoder-core/app/translator/platforms/sentinel_one/mapping.py b/uncoder-core/app/translator/platforms/sentinel_one/mapping.py new file mode 100644 index 00000000..f782551f --- /dev/null +++ b/uncoder-core/app/translator/platforms/sentinel_one/mapping.py @@ -0,0 +1,20 @@ +from app.translator.core.mapping import BasePlatformMappings, LogSourceSignature +from app.translator.platforms.sentinel_one.const import sentinel_one_power_query_details + + +class SentinelOnePowerQueryLogSourceSignature(LogSourceSignature): + def is_suitable(self) -> bool: + return True + + def __str__(self) -> str: + return "" + + +class SentinelOnePowerQueryMappings(BasePlatformMappings): + def prepare_log_source_signature(self, mapping: dict) -> SentinelOnePowerQueryLogSourceSignature: + ... + + +sentinel_one_power_query_query_mappings = SentinelOnePowerQueryMappings( + platform_dir="sentinel_one", platform_details=sentinel_one_power_query_details +) diff --git a/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py b/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py new file mode 100644 index 00000000..e79bef0d --- /dev/null +++ b/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py @@ -0,0 +1,110 @@ +from typing import Union, Optional + +from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.custom_types.values import ValueType +from app.translator.core.mapping import LogSourceSignature +from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender +from app.translator.core.str_value_manager import StrValueManager +from app.translator.managers import render_manager +from app.translator.platforms.sentinel_one.const import sentinel_one_power_query_details +from app.translator.platforms.sentinel_one.mapping import ( + SentinelOnePowerQueryMappings, + sentinel_one_power_query_query_mappings, +) +from app.translator.platforms.sentinel_one.str_value_manager import sentinel_one_power_query_str_value_manager + + +class SentinelOnePowerQueryFieldValue(BaseFieldValueRender): + details: PlatformDetails = sentinel_one_power_query_details + str_value_manager: StrValueManager = sentinel_one_power_query_str_value_manager + list_token = ", " + + @staticmethod + def _wrap_str_value(value: str) -> str: + return f'"{value}"' + + def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.list_token.join( + self._pre_process_value(field, v, value_type=ValueType.value, wrap_str=True) for v in value + ) + return f"{field} in ({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True) + return f"{field} = {value}" + + def less_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} < {value}" + + def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} <= {value}" + + def greater_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} > {value}" + + def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} >= {value}" + + def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.list_token.join( + self._pre_process_value(field, v, value_type=ValueType.value, wrap_str=True, wrap_int=True) + for v in value + ) + return f"{field} != ({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + return f"{field} != {value}" + + def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.list_token.join( + self._pre_process_value(field, v, value_type=ValueType.value, wrap_str=True, wrap_int=True) + for v in value + ) + return f"{field} contains ({values})" + value = self._pre_process_value(field, value, value_type=ValueType.value) + return f"{field} contains {value}" + + def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return self.contains_modifier(field, value) + + def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + return self.contains_modifier(field, value) + + def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: + if isinstance(value, list): + values = self.list_token.join( + self.str_value_manager.escape_manager.escape( + self._pre_process_value(field, v, value_type=ValueType.regex_value, wrap_str=True, wrap_int=True), + ValueType.regex_value, + ) + for v in value + ) + return f"{field} matches ({values})" + value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=True, wrap_int=True) + value = self.str_value_manager.escape_manager.escape(value, ValueType.regex_value) + return f"{field} matches {value}" + + def is_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 + return f'not ({field} matches "\\.*")' + + def is_not_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 + return f'{field} matches "\\.*"' + + +@render_manager.register +class SentinelOnePowerQueryRender(PlatformQueryRender): + details: PlatformDetails = sentinel_one_power_query_details + mappings: SentinelOnePowerQueryMappings = sentinel_one_power_query_query_mappings + or_token = "or" + and_token = "and" + not_token = "not" + comment_symbol = "//" + field_value_render = SentinelOnePowerQueryFieldValue(or_token=or_token) + + def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: + return "| columns " diff --git a/uncoder-core/app/translator/platforms/sentinel_one/str_value_manager.py b/uncoder-core/app/translator/platforms/sentinel_one/str_value_manager.py new file mode 100644 index 00000000..8c9a9341 --- /dev/null +++ b/uncoder-core/app/translator/platforms/sentinel_one/str_value_manager.py @@ -0,0 +1,31 @@ +""" +Uncoder IO Community Edition License +----------------------------------------------------------------- +Copyright (c) 2024 SOC Prime, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------- +""" + +from app.translator.core.str_value_manager import StrValueManager +from app.translator.platforms.sentinel_one.escape_manager import ( + SentinelOnePowerQueryEscapeManager, + sentinel_one_power_query_escape_manager, +) + + +class SentinelOnePowerQueryStrValueManager(StrValueManager): + escape_manager: SentinelOnePowerQueryEscapeManager = sentinel_one_power_query_escape_manager + + +sentinel_one_power_query_str_value_manager = SentinelOnePowerQueryStrValueManager() From 3c6c43f241417fd5c88f3e2105e376230a9bde6b Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:05:36 +0300 Subject: [PATCH 5/9] gis-8825 added sentinel one power query render --- .../platforms/sentinel_one/renders/sentinel_one_power_query.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py b/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py index e79bef0d..5c98437d 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py @@ -105,6 +105,3 @@ class SentinelOnePowerQueryRender(PlatformQueryRender): not_token = "not" comment_symbol = "//" field_value_render = SentinelOnePowerQueryFieldValue(or_token=or_token) - - def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: - return "| columns " From 73914f5be53fccd067e43d065da162189e822a90 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:05:51 +0300 Subject: [PATCH 6/9] gis-8825 fix --- .../platforms/sentinel_one/renders/sentinel_one_power_query.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py b/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py index 5c98437d..e3af9bdd 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py @@ -1,8 +1,7 @@ -from typing import Union, Optional +from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValueManager From 1ca0bb32b3dd0104983b6940af72782ca37c2720 Mon Sep 17 00:00:00 2001 From: Gesyk Nazar <77268518+nazargesyk@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:37:33 +0300 Subject: [PATCH 7/9] gis-8825 fixes --- .../sentinel_one/custom_types/__init__.py | 0 .../sentinel_one/custom_types/values.py | 5 +++++ .../platforms/sentinel_one/escape_manager.py | 4 +++- .../renders/sentinel_one_power_query.py | 14 +++++--------- .../platforms/sentinel_one/str_value_manager.py | 17 +++++++++++++++-- 5 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 uncoder-core/app/translator/platforms/sentinel_one/custom_types/__init__.py create mode 100644 uncoder-core/app/translator/platforms/sentinel_one/custom_types/values.py diff --git a/uncoder-core/app/translator/platforms/sentinel_one/custom_types/__init__.py b/uncoder-core/app/translator/platforms/sentinel_one/custom_types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/uncoder-core/app/translator/platforms/sentinel_one/custom_types/values.py b/uncoder-core/app/translator/platforms/sentinel_one/custom_types/values.py new file mode 100644 index 00000000..c009aa9a --- /dev/null +++ b/uncoder-core/app/translator/platforms/sentinel_one/custom_types/values.py @@ -0,0 +1,5 @@ +from app.translator.core.custom_types.values import ValueType + + +class SentinelOneValueType(ValueType): + double_escape_regex_value = "d_e_re_value" diff --git a/uncoder-core/app/translator/platforms/sentinel_one/escape_manager.py b/uncoder-core/app/translator/platforms/sentinel_one/escape_manager.py index 45232ef1..04193dce 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/escape_manager.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/escape_manager.py @@ -3,12 +3,14 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager from app.translator.core.models.escape_details import EscapeDetails +from app.translator.platforms.sentinel_one.custom_types.values import SentinelOneValueType class SentinelOnePowerQueryEscapeManager(EscapeManager): escape_map: ClassVar[dict[str, list[EscapeDetails]]] = { ValueType.value: [EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\")], - ValueType.regex_value: [EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\")], + ValueType.regex_value: [EscapeDetails(pattern=r"([$^*+()\[\]{}|.?\-\\])", escape_symbols=r"\\\1")], + SentinelOneValueType.double_escape_regex_value: [EscapeDetails(pattern=r"\\", escape_symbols=r"\\\\")], } diff --git a/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py b/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py index e3af9bdd..0e827722 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/renders/sentinel_one_power_query.py @@ -33,19 +33,19 @@ def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"{field} = {value}" def less_modifier(self, field: str, value: Union[int, str]) -> str: - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True) return f"{field} < {value}" def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True) return f"{field} <= {value}" def greater_modifier(self, field: str, value: Union[int, str]) -> str: - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True) return f"{field} > {value}" def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str: - value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True, wrap_int=True) + value = self._pre_process_value(field, value, value_type=ValueType.value, wrap_str=True) return f"{field} >= {value}" def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -77,15 +77,11 @@ def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): values = self.list_token.join( - self.str_value_manager.escape_manager.escape( - self._pre_process_value(field, v, value_type=ValueType.regex_value, wrap_str=True, wrap_int=True), - ValueType.regex_value, - ) + self._pre_process_value(field, v, value_type=ValueType.regex_value, wrap_str=True, wrap_int=True) for v in value ) return f"{field} matches ({values})" value = self._pre_process_value(field, value, value_type=ValueType.regex_value, wrap_str=True, wrap_int=True) - value = self.str_value_manager.escape_manager.escape(value, ValueType.regex_value) return f"{field} matches {value}" def is_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002 diff --git a/uncoder-core/app/translator/platforms/sentinel_one/str_value_manager.py b/uncoder-core/app/translator/platforms/sentinel_one/str_value_manager.py index 8c9a9341..3a48457a 100644 --- a/uncoder-core/app/translator/platforms/sentinel_one/str_value_manager.py +++ b/uncoder-core/app/translator/platforms/sentinel_one/str_value_manager.py @@ -16,8 +16,9 @@ limitations under the License. ----------------------------------------------------------------- """ - -from app.translator.core.str_value_manager import StrValueManager +from app.translator.core.custom_types.values import ValueType +from app.translator.core.str_value_manager import BaseSpecSymbol, StrValue, StrValueManager +from app.translator.platforms.sentinel_one.custom_types.values import SentinelOneValueType from app.translator.platforms.sentinel_one.escape_manager import ( SentinelOnePowerQueryEscapeManager, sentinel_one_power_query_escape_manager, @@ -27,5 +28,17 @@ class SentinelOnePowerQueryStrValueManager(StrValueManager): escape_manager: SentinelOnePowerQueryEscapeManager = sentinel_one_power_query_escape_manager + def from_container_to_str(self, container: StrValue, value_type: str = ValueType.value) -> str: + result = "" + for el in container.split_value: + if isinstance(el, str): + result += self.escape_manager.escape(el, value_type) + elif isinstance(el, BaseSpecSymbol) and (pattern := self.container_spec_symbols_map.get(type(el))): + if value_type == ValueType.regex_value: + pattern = self.escape_manager.escape(pattern, SentinelOneValueType.double_escape_regex_value) + result += pattern + + return result + sentinel_one_power_query_str_value_manager = SentinelOnePowerQueryStrValueManager() From 73dee61e6bcd08be46958334d0b768b540d509c3 Mon Sep 17 00:00:00 2001 From: Nazar Gesyk Date: Tue, 17 Dec 2024 16:08:59 +0200 Subject: [PATCH 8/9] gis-8825 cleaning --- uncoder-core/app/translator/core/parser.py | 10 +--------- .../app/translator/platforms/base/aql/parsers/aql.py | 4 ++-- .../translator/platforms/base/lucene/parsers/lucene.py | 8 +++----- .../app/translator/platforms/base/sql/parsers/sql.py | 7 +++---- .../platforms/microsoft/parsers/microsoft_sentinel.py | 3 +-- 5 files changed, 10 insertions(+), 22 deletions(-) diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 2f632b4e..da7330eb 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -24,7 +24,7 @@ from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping -from app.translator.core.models.functions.base import Function, ParsedFunctions +from app.translator.core.models.functions.base import Function from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.models.query_tokens.field import Field @@ -51,9 +51,6 @@ def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: raise NotImplementedError("Abstract method") - def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: - raise NotImplementedError("Abstract method") - class PlatformQueryParser(QueryParser, ABC): mappings: BasePlatformMappings = None @@ -91,8 +88,3 @@ def get_source_mappings( ) self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping) return source_mappings - - def get_source_mapping_ids_by_logsources(self, query: str) -> Optional[list[str]]: - _, parsed_logsources, _ = self._parse_query(query=query) - if parsed_logsources: - return self.mappings.get_source_mappings_by_log_sources(parsed_logsources) diff --git a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py index 44800cf9..509b1545 100644 --- a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py @@ -17,7 +17,7 @@ """ import re -from typing import Optional, Union +from typing import Union from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.models.functions.base import ParsedFunctions @@ -105,7 +105,7 @@ def __parse_log_sources(self, query: str) -> tuple[dict[str, Union[list[str], li return log_sources, query - def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[int]]], ParsedFunctions]: query = self.__clean_query(query) self.__check_table(query) query, functions = self.platform_functions.parse(query) diff --git a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py index 77ef79f4..49f05c98 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/parsers/lucene.py @@ -17,9 +17,7 @@ """ import re -from typing import Optional, Union -from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.lucene.tokenizer import LuceneTokenizer @@ -33,7 +31,7 @@ class LuceneQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: log_sources = {} for source_type in self.log_source_key_types: pattern = self.log_source_pattern.replace("___source_type___", source_type) @@ -45,10 +43,10 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list pos_end = search.end() query = query[:pos_start] + query[pos_end:] - return query, log_sources, None + return query, log_sources def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - query, log_sources, _ = self._parse_query(raw_query_container.query) + query, log_sources = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) source_mappings = self.get_source_mappings(query_field_tokens, log_sources) diff --git a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py index 2b5854cb..1e91832b 100644 --- a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py @@ -17,7 +17,6 @@ """ import re -from typing import Optional, Union from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer @@ -32,7 +31,7 @@ class SqlQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*--.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: log_source = {"table": []} if re.search(self.query_delimiter_pattern, query, flags=re.IGNORECASE): table_search = re.search(self.table_pattern, query) @@ -40,10 +39,10 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list log_source["table"] = [table] return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source, None - return query, log_source, None + return query, log_source def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: - query, log_sources, _ = self._parse_query(raw_query_container.query) + query, log_sources = self._parse_query(raw_query_container.query) query_tokens = self.get_query_tokens(query) query_field_tokens, _, _ = self.get_field_tokens(query_tokens) source_mappings = self.get_source_mappings(query_field_tokens, log_sources) diff --git a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index ecebd04b..e7392ea3 100644 --- a/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -16,7 +16,6 @@ ----------------------------------------------------------------- """ -from typing import Optional, Union from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.platform_details import PlatformDetails @@ -38,7 +37,7 @@ class MicrosoftSentinelQueryParser(PlatformQueryParser): wrapped_with_comment_pattern = r"^\s*//.*(?:\n|$)" - def _parse_query(self, query: str) -> tuple[str, dict[str, Union[list[str], list[int]]], Optional[ParsedFunctions]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFunctions]: table, query, functions = self.platform_functions.parse(query) log_sources = {"table": [table]} return query, log_sources, functions From ba9ee98a9dffabfcf187a98b29867e6a3fb84e7b Mon Sep 17 00:00:00 2001 From: Nazar Gesyk Date: Tue, 17 Dec 2024 16:11:18 +0200 Subject: [PATCH 9/9] gis-8825 cleaning --- uncoder-core/app/translator/platforms/base/aql/parsers/aql.py | 2 +- uncoder-core/app/translator/platforms/base/sql/parsers/sql.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py index 509b1545..0dad8283 100644 --- a/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/parsers/aql.py @@ -106,7 +106,7 @@ def __parse_log_sources(self, query: str) -> tuple[dict[str, Union[list[str], li return log_sources, query def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[int]]], ParsedFunctions]: - query = self.__clean_query(query) + query = self.__clean_query(text) self.__check_table(query) query, functions = self.platform_functions.parse(query) log_sources, query = self.__parse_log_sources(query) diff --git a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py index 1e91832b..01be3500 100644 --- a/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/parsers/sql.py @@ -18,7 +18,6 @@ import re -from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.sql.tokenizer import SqlTokenizer @@ -37,7 +36,7 @@ def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: table_search = re.search(self.table_pattern, query) table = table_search.group("table") log_source["table"] = [table] - return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source, None + return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source return query, log_source pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy