Skip to content

IOC`s fix mapping and hash types #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 14, 2023
Merged

IOC`s fix mapping and hash types #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
DEFAULT_GRAYLOG_MAPPING = {
"SourceIP": "sourceAddress",
"DestinationIP": "destinationAddress",
"Domain": "destinationDnsDomain",
"URL": "requestUrl",
"HashMd5": "fileHash",
"HashSha1": "fileHash",
"HashSha256": "fileHash",
"HashSha512": "fileHash",
"SourceIP": "source.ip",
"DestinationIP": "destination.ip",
"Domain": "destination.domain",
"URL": "url.original",
"HashMd5": "file.hash.md5",
"HashSha1": "file.hash.sha1",
"HashSha256": "file.hash.sha256",
"HashSha512": "file.hash.sha512",
"Emails": "emails",
"Files": "filePath"
}
4 changes: 2 additions & 2 deletions siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
DEFAULT_MICROSOFT_SENTINEL_MAPPING = {
"DestinationIP": "DestinationIP",
"DestinationIP": "DestinationIp",
"SourceIP": "SourceIp",
"HashSha512": "FileHashSha512",
"HashSha256": "FileHashSha256",
"HashMd5": "FileHashMd5",
"Emails": "SenderFromAddress",
"Domain": "DestinationHostname",
"HashSha1": "FileHashSha1",
"Files": "FileName",
"Files": "TargetFileName",
"URL": "URL"
}
26 changes: 17 additions & 9 deletions siem-converter/app/converter/core/parser_cti.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,29 @@
from pydantic import BaseModel

from app.converter.core.exceptions.iocs import IocsLimitExceededException, EmptyIOCSException
from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN,\
hash_regexes, IOCType, HashType, IocParsingRule
from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN, \
hash_regexes, IOCType, HashType, IocParsingRule, HASH_MAP


class Iocs(BaseModel):
ip: list[str] = []
url: list[str] = []
domain: list[str] = []
hash: list[str] = []
hash_dict: dict = {}

def get_total_count(self) -> int:
return len(self.ip) + len(self.url) + len(self.domain) + len(self.hash)
hash_len = 0
for value in self.hash_dict.values():
hash_len += len(value)
return len(self.ip) + len(self.url) + len(self.domain) + hash_len

def return_iocs(self) -> dict:
if all(not value for value in [self.ip, self.url, self.domain, self.hash]):
if all(not value for value in [self.ip, self.url, self.domain, self.hash_dict]):
raise EmptyIOCSException()
return {"ip": self.ip, "url": self.url, "domain": self.domain, "hash": self.hash}
result = {"DestinationIP": self.ip, "URL": self.url, "Domain": self.domain}
for key, value in self.hash_dict.items():
result[HASH_MAP[key]] = value
return result


class CTIParser:
Expand All @@ -47,7 +53,7 @@ def get_iocs_from_string(
if not include_hash_types:
include_hash_types = list(hash_regexes.keys())
for hash_type in include_hash_types:
iocs.hash.extend(self._find_all_str_by_regex(string, hash_regexes[hash_type]))
iocs.hash_dict[hash_type] = self._find_all_str_by_regex(string, hash_regexes[hash_type])
iocs = self.remove_duplicates(iocs)
iocs = self.remove_exceptions(iocs, exceptions)
if ioc_parsing_rules is None or "remove_private_and_reserved_ips" in ioc_parsing_rules:
Expand All @@ -69,14 +75,16 @@ def remove_duplicates(self, iocs):
iocs.ip = self._remove_duplicates_from_list(iocs.ip)
iocs.domain = self._remove_duplicates_from_list(iocs.domain)
iocs.url = self._remove_duplicates_from_list(iocs.url)
iocs.hash = self._remove_duplicates_from_list(iocs.hash)
for key, value in iocs.hash_dict.items():
iocs.hash_dict[key] = self._remove_duplicates_from_list(value)
return iocs

def remove_exceptions(self, iocs, exceptions=None):
iocs.ip = self._remove_exceptions(iocs.ip, exceptions)
iocs.domain = self._remove_exceptions(iocs.domain, exceptions)
iocs.url = self._remove_exceptions(iocs.url, exceptions)
iocs.hash = self._remove_exceptions(iocs.hash, exceptions)
for key, value in iocs.hash_dict.items():
iocs.hash_dict[key] = self._remove_exceptions(value, exceptions)
return iocs

@staticmethod
Expand Down
29 changes: 4 additions & 25 deletions siem-converter/app/converter/core/render_cti.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,6 @@ class RenderCTI:
final_result_for_one: str = "union * | where {result}\n"
default_mapping = None

def get_default_mapping(self, include_source_ip=False):
return self.prepare_mapping(self.default_mapping, include_source_ip)

def create_field_value(self, field: str, value: str, generic_field: str):
return self.data_map.format(key=field, value=value)

Expand All @@ -53,35 +50,17 @@ def render(self, data: List[List[IocsChunkValue]]) -> list[str]:
def collect_data_values(self, chunk):
data_values = []
key_chunk = []
processing_key = chunk[0].generic_field
processing_key = chunk[0].platform_field
for value in chunk:
if processing_key != value.generic_field:
if processing_key != value.platform_field:
data_values.append(self.or_group.format(or_group=self.or_operator.join(key_chunk),
processing_key=processing_key))
key_chunk = []
processing_key = value.generic_field
key_chunk.append(self.create_field_value(field=value.generic_field,
processing_key = value.platform_field
key_chunk.append(self.create_field_value(field=value.platform_field,
value=value.value,
generic_field=value.generic_field))
if key_chunk:
data_values.append(
self.or_group.format(or_group=self.or_operator.join(key_chunk), processing_key=processing_key))
return data_values

def prepare_mapping(self, mapping: dict, include_source_ip: bool = False) -> dict:
m = {
"DestinationIP": "ip",
"Domain": "domain",
"URL": "url",
"HashMd5": "hash",
"HashSha1": "hash",
"HashSha256": "hash",
"HashSha512": "hash",
}
if include_source_ip:
m["SourceIP"] = "ip"
res = {}
for key, new_field_name in mapping.items():
if key in m:
res[new_field_name] = m[key]
return res
22 changes: 8 additions & 14 deletions siem-converter/app/converter/cti_converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,8 @@ def __init__(self):
self.logger = logging.getLogger("cti_converter")
self.parser = CTIParser()

def _reverse_mapping(self, platform: CTIPlatform, data: Dict[str, List[str]],
include_source_ip: bool = False) -> Dict[str, str]:
mapping: Dict = self.renders.get(platform.name).get_default_mapping(include_source_ip)
reverse_mapping = {}
for platform_field, generic_field in mapping.items():
if data.get(generic_field):
reverse_mapping.update({generic_field: platform_field})
return reverse_mapping
def _get_render_mapping(self, platform: CTIPlatform, include_source_ip: bool = False) -> Dict[str, str]:
return self.renders.get(platform.name).default_mapping

@handle_translation_exceptions
def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, include_hash_types: list = None,
Expand All @@ -39,11 +33,10 @@ def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, in
@handle_translation_exceptions
def __render_translation(self, parsed_data: dict, platform_data: CTIPlatform, iocs_per_query: int,
include_source_ip: bool = False) -> List[str]:
reverse_mapping = self._reverse_mapping(data=parsed_data,
include_source_ip=include_source_ip, platform=platform_data)
mapping = self._get_render_mapping(platform=platform_data, include_source_ip=include_source_ip)
platform = self.renders.get(platform_data.name)
platform_generation = self.generate(data=parsed_data, platform=platform, iocs_per_query=iocs_per_query,
mapping=reverse_mapping)
mapping=mapping)
return platform_generation

def convert(self, text: str,
Expand Down Expand Up @@ -73,9 +66,10 @@ def _get_iocs_chunk(chunks_size: int, data: Dict[str, List[str]],
result = []
for generic_field, iocs_list in data.items():
for ioc in iocs_list:
result.append(IocsChunkValue(generic_field=generic_field,
platform_field=mapping.get(generic_field, generic_field),
value=ioc))
if mapping.get(generic_field):
result.append(IocsChunkValue(generic_field=generic_field,
platform_field=mapping[generic_field],
value=ioc))
return [result[i:i + chunks_size] for i in range(0, len(result), chunks_size)]

def generate(self, platform: RenderCTI, iocs_per_query, data: Dict[str, List[str]],
Expand Down
7 changes: 7 additions & 0 deletions siem-converter/app/converter/tools/const.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@
"replace_dots", "remove_private_and_reserved_ips", "replace_hxxp"
]

HASH_MAP = {
"md5": "HashMd5",
"sha1": "HashSha1",
"sha256": "HashSha256",
"sha512": "HashSha512"
}

hash_regexes = {
"md5": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{32})(?=[\s)\]\"',;\n<|]|$)",
"sha1": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{40})(?=[\s)\]\"',;\n<|]|$)",
Expand Down
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy