From 35587faeffe1895bdcb53a17a89611c4850089c3 Mon Sep 17 00:00:00 2001 From: "nazar.gesyk" Date: Tue, 14 Nov 2023 12:22:40 +0200 Subject: [PATCH] IOC`s fix mapping and hash types --- .../backends/graylog/mappings/graylog_cti.py | 16 +++++----- .../mappings/microsoft_sentinel_cti.py | 4 +-- .../app/converter/core/parser_cti.py | 26 +++++++++++------ .../app/converter/core/render_cti.py | 29 +++---------------- siem-converter/app/converter/cti_converter.py | 22 +++++--------- siem-converter/app/converter/tools/const.py | 7 +++++ 6 files changed, 46 insertions(+), 58 deletions(-) diff --git a/siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py b/siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py index f2f32690..eb66315b 100644 --- a/siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py +++ b/siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py @@ -1,12 +1,12 @@ DEFAULT_GRAYLOG_MAPPING = { - "SourceIP": "sourceAddress", - "DestinationIP": "destinationAddress", - "Domain": "destinationDnsDomain", - "URL": "requestUrl", - "HashMd5": "fileHash", - "HashSha1": "fileHash", - "HashSha256": "fileHash", - "HashSha512": "fileHash", + "SourceIP": "source.ip", + "DestinationIP": "destination.ip", + "Domain": "destination.domain", + "URL": "url.original", + "HashMd5": "file.hash.md5", + "HashSha1": "file.hash.sha1", + "HashSha256": "file.hash.sha256", + "HashSha512": "file.hash.sha512", "Emails": "emails", "Files": "filePath" } diff --git a/siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py b/siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py index 8f9ac3ea..f2cf3a9d 100644 --- a/siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py +++ b/siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py @@ -1,5 +1,5 @@ DEFAULT_MICROSOFT_SENTINEL_MAPPING = { - "DestinationIP": "DestinationIP", + "DestinationIP": "DestinationIp", "SourceIP": "SourceIp", "HashSha512": "FileHashSha512", "HashSha256": "FileHashSha256", @@ -7,6 +7,6 @@ "Emails": "SenderFromAddress", "Domain": "DestinationHostname", "HashSha1": "FileHashSha1", - "Files": "FileName", + "Files": "TargetFileName", "URL": "URL" } diff --git a/siem-converter/app/converter/core/parser_cti.py b/siem-converter/app/converter/core/parser_cti.py index ff173df1..4d33fe31 100644 --- a/siem-converter/app/converter/core/parser_cti.py +++ b/siem-converter/app/converter/core/parser_cti.py @@ -5,23 +5,29 @@ from pydantic import BaseModel from app.converter.core.exceptions.iocs import IocsLimitExceededException, EmptyIOCSException -from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN,\ - hash_regexes, IOCType, HashType, IocParsingRule +from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN, \ + hash_regexes, IOCType, HashType, IocParsingRule, HASH_MAP class Iocs(BaseModel): ip: list[str] = [] url: list[str] = [] domain: list[str] = [] - hash: list[str] = [] + hash_dict: dict = {} def get_total_count(self) -> int: - return len(self.ip) + len(self.url) + len(self.domain) + len(self.hash) + hash_len = 0 + for value in self.hash_dict.values(): + hash_len += len(value) + return len(self.ip) + len(self.url) + len(self.domain) + hash_len def return_iocs(self) -> dict: - if all(not value for value in [self.ip, self.url, self.domain, self.hash]): + if all(not value for value in [self.ip, self.url, self.domain, self.hash_dict]): raise EmptyIOCSException() - return {"ip": self.ip, "url": self.url, "domain": self.domain, "hash": self.hash} + result = {"DestinationIP": self.ip, "URL": self.url, "Domain": self.domain} + for key, value in self.hash_dict.items(): + result[HASH_MAP[key]] = value + return result class CTIParser: @@ -47,7 +53,7 @@ def get_iocs_from_string( if not include_hash_types: include_hash_types = list(hash_regexes.keys()) for hash_type in include_hash_types: - iocs.hash.extend(self._find_all_str_by_regex(string, hash_regexes[hash_type])) + iocs.hash_dict[hash_type] = self._find_all_str_by_regex(string, hash_regexes[hash_type]) iocs = self.remove_duplicates(iocs) iocs = self.remove_exceptions(iocs, exceptions) if ioc_parsing_rules is None or "remove_private_and_reserved_ips" in ioc_parsing_rules: @@ -69,14 +75,16 @@ def remove_duplicates(self, iocs): iocs.ip = self._remove_duplicates_from_list(iocs.ip) iocs.domain = self._remove_duplicates_from_list(iocs.domain) iocs.url = self._remove_duplicates_from_list(iocs.url) - iocs.hash = self._remove_duplicates_from_list(iocs.hash) + for key, value in iocs.hash_dict.items(): + iocs.hash_dict[key] = self._remove_duplicates_from_list(value) return iocs def remove_exceptions(self, iocs, exceptions=None): iocs.ip = self._remove_exceptions(iocs.ip, exceptions) iocs.domain = self._remove_exceptions(iocs.domain, exceptions) iocs.url = self._remove_exceptions(iocs.url, exceptions) - iocs.hash = self._remove_exceptions(iocs.hash, exceptions) + for key, value in iocs.hash_dict.items(): + iocs.hash_dict[key] = self._remove_exceptions(value, exceptions) return iocs @staticmethod diff --git a/siem-converter/app/converter/core/render_cti.py b/siem-converter/app/converter/core/render_cti.py index daf18a51..bf8d2c68 100644 --- a/siem-converter/app/converter/core/render_cti.py +++ b/siem-converter/app/converter/core/render_cti.py @@ -34,9 +34,6 @@ class RenderCTI: final_result_for_one: str = "union * | where {result}\n" default_mapping = None - def get_default_mapping(self, include_source_ip=False): - return self.prepare_mapping(self.default_mapping, include_source_ip) - def create_field_value(self, field: str, value: str, generic_field: str): return self.data_map.format(key=field, value=value) @@ -53,35 +50,17 @@ def render(self, data: List[List[IocsChunkValue]]) -> list[str]: def collect_data_values(self, chunk): data_values = [] key_chunk = [] - processing_key = chunk[0].generic_field + processing_key = chunk[0].platform_field for value in chunk: - if processing_key != value.generic_field: + if processing_key != value.platform_field: data_values.append(self.or_group.format(or_group=self.or_operator.join(key_chunk), processing_key=processing_key)) key_chunk = [] - processing_key = value.generic_field - key_chunk.append(self.create_field_value(field=value.generic_field, + processing_key = value.platform_field + key_chunk.append(self.create_field_value(field=value.platform_field, value=value.value, generic_field=value.generic_field)) if key_chunk: data_values.append( self.or_group.format(or_group=self.or_operator.join(key_chunk), processing_key=processing_key)) return data_values - - def prepare_mapping(self, mapping: dict, include_source_ip: bool = False) -> dict: - m = { - "DestinationIP": "ip", - "Domain": "domain", - "URL": "url", - "HashMd5": "hash", - "HashSha1": "hash", - "HashSha256": "hash", - "HashSha512": "hash", - } - if include_source_ip: - m["SourceIP"] = "ip" - res = {} - for key, new_field_name in mapping.items(): - if key in m: - res[new_field_name] = m[key] - return res diff --git a/siem-converter/app/converter/cti_converter.py b/siem-converter/app/converter/cti_converter.py index 7a9a4ed6..a0070b1f 100644 --- a/siem-converter/app/converter/cti_converter.py +++ b/siem-converter/app/converter/cti_converter.py @@ -17,14 +17,8 @@ def __init__(self): self.logger = logging.getLogger("cti_converter") self.parser = CTIParser() - def _reverse_mapping(self, platform: CTIPlatform, data: Dict[str, List[str]], - include_source_ip: bool = False) -> Dict[str, str]: - mapping: Dict = self.renders.get(platform.name).get_default_mapping(include_source_ip) - reverse_mapping = {} - for platform_field, generic_field in mapping.items(): - if data.get(generic_field): - reverse_mapping.update({generic_field: platform_field}) - return reverse_mapping + def _get_render_mapping(self, platform: CTIPlatform, include_source_ip: bool = False) -> Dict[str, str]: + return self.renders.get(platform.name).default_mapping @handle_translation_exceptions def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, include_hash_types: list = None, @@ -39,11 +33,10 @@ def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, in @handle_translation_exceptions def __render_translation(self, parsed_data: dict, platform_data: CTIPlatform, iocs_per_query: int, include_source_ip: bool = False) -> List[str]: - reverse_mapping = self._reverse_mapping(data=parsed_data, - include_source_ip=include_source_ip, platform=platform_data) + mapping = self._get_render_mapping(platform=platform_data, include_source_ip=include_source_ip) platform = self.renders.get(platform_data.name) platform_generation = self.generate(data=parsed_data, platform=platform, iocs_per_query=iocs_per_query, - mapping=reverse_mapping) + mapping=mapping) return platform_generation def convert(self, text: str, @@ -73,9 +66,10 @@ def _get_iocs_chunk(chunks_size: int, data: Dict[str, List[str]], result = [] for generic_field, iocs_list in data.items(): for ioc in iocs_list: - result.append(IocsChunkValue(generic_field=generic_field, - platform_field=mapping.get(generic_field, generic_field), - value=ioc)) + if mapping.get(generic_field): + result.append(IocsChunkValue(generic_field=generic_field, + platform_field=mapping[generic_field], + value=ioc)) return [result[i:i + chunks_size] for i in range(0, len(result), chunks_size)] def generate(self, platform: RenderCTI, iocs_per_query, data: Dict[str, List[str]], diff --git a/siem-converter/app/converter/tools/const.py b/siem-converter/app/converter/tools/const.py index 611083d9..1616189a 100644 --- a/siem-converter/app/converter/tools/const.py +++ b/siem-converter/app/converter/tools/const.py @@ -10,6 +10,13 @@ "replace_dots", "remove_private_and_reserved_ips", "replace_hxxp" ] +HASH_MAP = { + "md5": "HashMd5", + "sha1": "HashSha1", + "sha256": "HashSha256", + "sha512": "HashSha512" +} + hash_regexes = { "md5": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{32})(?=[\s)\]\"',;\n<|]|$)", "sha1": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{40})(?=[\s)\]\"',;\n<|]|$)", pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy