Skip to content

Commit 0236ff1

Browse files
nazargesyknazargesyk
committed
gis-9379 add new endpoint /iocs/generate
1 parent c868b92 commit 0236ff1

File tree

7 files changed

+209
-10
lines changed

7 files changed

+209
-10
lines changed

‎uncoder-core/app/routers/ioc_translate.py

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,10 @@
44

55
from app.models.ioc_translation import CTIPlatform, OneTranslationCTIData
66
from app.models.translation import InfoMessage
7-
from app.translator.cti_translator import CTITranslator
7+
from app.translator.cti_translator import cti_translator
88
from app.translator.tools.const import HashType, IocParsingRule, IOCType
99

1010
iocs_router = APIRouter()
11-
cti_translator = CTITranslator()
1211

1312

1413
@iocs_router.post("/iocs/translate", description="Parse IOCs from text.")
@@ -46,3 +45,32 @@ def parse_and_translate_iocs(
4645

4746
info_message = InfoMessage(message=translations, severity="error")
4847
return OneTranslationCTIData(info=info_message, status=status, target_platform_id=platform.id)
48+
49+
50+
@iocs_router.post("/iocs/generate", description="Parse IOCs from text and based on input data generate translation")
51+
@iocs_router.post("/iocs/generate", include_in_schema=False)
52+
def parse_iocs_and_generate_rule(
53+
text: str = Body(..., description="Text to parse IOCs from", embed=True),
54+
platform: CTIPlatform = Body(..., description="Platform to parse IOCs to", embed=True),
55+
iocs_per_query: int = Body(25, description="IOCs per query limit", embed=True),
56+
title: str = Body(..., description="Title", embed=True),
57+
description: str = Body(..., description="Description", embed=True),
58+
references: list[str] = Body(..., description="References", embed=True),
59+
created_date: str = Body(..., description="Rule created date", embed=True),
60+
mitre_tags: Optional[list[str]] = Body(..., description="Mitra tactics and techniques", embed=True),
61+
) -> OneTranslationCTIData:
62+
status, translations = cti_translator.generate(
63+
title=title,
64+
text=text,
65+
platform_data=platform,
66+
description=description,
67+
references=references,
68+
created_date=created_date,
69+
mitre_tags=mitre_tags,
70+
iocs_per_query=iocs_per_query,
71+
)
72+
if status:
73+
return OneTranslationCTIData(status=status, translations=translations, target_platform_id=platform.id)
74+
75+
info_message = InfoMessage(message=translations, severity="error")
76+
return OneTranslationCTIData(info=info_message, status=status, target_platform_id=platform.id)

‎uncoder-core/app/translator/core/mixins/tokens.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
from app.translator.core.const import QUERY_TOKEN_TYPE
2+
from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType
3+
from app.translator.core.mapping import SourceMapping
4+
from app.translator.core.models.query_tokens.field_value import FieldValue
5+
from app.translator.core.models.query_tokens.identifier import Identifier
6+
7+
8+
class ExtraConditionMixin:
9+
def generate_extra_conditions(self, source_mapping: SourceMapping) -> list[QUERY_TOKEN_TYPE]:
10+
extra_tokens = []
11+
for field, value in source_mapping.conditions.items():
12+
extra_tokens.extend(
13+
[
14+
FieldValue(source_name=field, operator=Identifier(token_type=OperatorType.EQ), value=value),
15+
Identifier(token_type=LogicalOperatorType.AND),
16+
]
17+
)
18+
return extra_tokens

‎uncoder-core/app/translator/core/render_cti.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
from abc import abstractmethod
1920

2021
from app.translator.core.models.iocs import IocsChunkValue
2122
from app.translator.core.models.platform_details import PlatformDetails
@@ -46,6 +47,10 @@ def render(self, data: list[list[IocsChunkValue]]) -> list[str]:
4647
final_result.append(self.final_result_for_one.format(result=data_values[0]))
4748
return final_result
4849

50+
@abstractmethod
51+
def generate(self, data: dict[str, list[list[IocsChunkValue]]], **kwargs) -> list[str]:
52+
raise NotImplementedError("Abstract method")
53+
4954
def collect_data_values(self, chunk: list[IocsChunkValue]) -> list[str]:
5055
data_values = []
5156
key_chunk = []

‎uncoder-core/app/translator/cti_translator.py

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from app.translator.core.models.iocs import IocsChunkValue
77
from app.translator.core.parser_cti import CTIParser
88
from app.translator.managers import RenderCTIManager, render_cti_manager
9+
from app.translator.tools.const import DefaultHashType, DefaultIocParsingRule, DefaultIOCType, iocs_types_map
910
from app.translator.tools.decorators import handle_translation_exceptions
1011

1112

@@ -45,6 +46,31 @@ def __render_translation(self, parsed_data: dict, platform_data: CTIPlatform, io
4546
)
4647
return render_cti.render(chunked_iocs)
4748

49+
def __sort_iocs_by_type(self, parsed_data: dict) -> dict:
50+
result = {}
51+
for key, values in iocs_types_map.items():
52+
if not result.get(key):
53+
result[key] = {}
54+
for generic_field, iocs_list in parsed_data.items():
55+
if generic_field in values:
56+
result[key][generic_field] = iocs_list
57+
return result
58+
59+
@handle_translation_exceptions
60+
def __generate_translation(
61+
self, parsed_data: dict, platform_data: CTIPlatform, iocs_per_query: int, **kwargs
62+
) -> list[str]:
63+
render_cti = self.render_manager.get(platform_data.id)
64+
65+
sorted_data = self.__sort_iocs_by_type(parsed_data)
66+
chunked_iocs = {}
67+
for key, chunk in sorted_data.items():
68+
if ioc_chuck := self.__get_iocs_chunk(
69+
chunks_size=iocs_per_query, data=chunk, mapping=render_cti.default_mapping
70+
):
71+
chunked_iocs[key] = ioc_chuck
72+
return render_cti.generate(chunked_iocs, **kwargs)
73+
4874
def translate(
4975
self,
5076
text: str,
@@ -70,6 +96,37 @@ def translate(
7096
)
7197
return status, parsed_data
7298

99+
def generate(
100+
self,
101+
text: str,
102+
title: str,
103+
description: str,
104+
references: list[str],
105+
created_date: str,
106+
mitre_tags: Optional[list[str]],
107+
platform_data: CTIPlatform,
108+
iocs_per_query: int = CTI_IOCS_PER_QUERY_LIMIT,
109+
) -> (bool, list[str]):
110+
status, parsed_data = self.__parse_iocs_from_string(
111+
text=text,
112+
include_ioc_types=DefaultIOCType,
113+
include_hash_types=DefaultHashType,
114+
ioc_parsing_rules=DefaultIocParsingRule,
115+
include_source_ip=True,
116+
)
117+
if status:
118+
kwargs = {
119+
"title": title,
120+
"description": description,
121+
"references": references,
122+
"created_date": created_date,
123+
"mitre_tags": mitre_tags,
124+
}
125+
return self.__generate_translation(
126+
parsed_data=parsed_data, platform_data=platform_data, iocs_per_query=iocs_per_query, **kwargs
127+
)
128+
return status, parsed_data
129+
73130
@staticmethod
74131
def __get_iocs_chunk(
75132
chunks_size: int, data: dict[str, list[str]], mapping: dict[str, str]
@@ -86,3 +143,6 @@ def __get_iocs_chunk(
86143
@classmethod
87144
def get_renders(cls) -> list:
88145
return cls.render_manager.get_platforms_details
146+
147+
148+
cti_translator = CTITranslator()

‎uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
11
from app.translator.core.models.platform_details import PlatformDetails
22
from app.translator.core.render_cti import RenderCTI
33
from app.translator.managers import render_cti_manager
4-
from app.translator.platforms.arcsight.const import ARCSIGHT_QUERY_DETAILS
5-
from app.translator.platforms.arcsight.mappings.arcsight_cti import DEFAULT_ARCSIGHT_MAPPING
4+
from app.translator.platforms.arcsight.const import DEFAULT_ARCSIGHT_CTI_MAPPING, arcsight_query_details
65

76

87
@render_cti_manager.register
98
class ArcsightKeyword(RenderCTI):
10-
details: PlatformDetails = PlatformDetails(**ARCSIGHT_QUERY_DETAILS)
9+
details: PlatformDetails = arcsight_query_details
1110

12-
default_mapping = DEFAULT_ARCSIGHT_MAPPING
11+
default_mapping = DEFAULT_ARCSIGHT_CTI_MAPPING
1312
field_value_template: str = "{key} = {value}"
1413
or_operator: str = " OR "
1514
group_or_operator: str = " OR "

‎uncoder-core/app/translator/platforms/sigma/renders/sigma_cti.py

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import uuid
2+
import yaml
3+
4+
from app.translator.core.custom_types.meta_info import SeverityType
5+
from app.translator.core.models.iocs import IocsChunkValue
6+
from app.translator.core.models.platform_details import PlatformDetails
7+
from app.translator.core.render_cti import RenderCTI
8+
from app.translator.managers import render_cti_manager
9+
from app.translator.platforms.sigma.const import sigma_rule_details, DEFAULT_SIGMA_CTI_MAPPING
10+
from app.translator.tools.const import LOGSOURCE_MAP
11+
12+
13+
@render_cti_manager.register
14+
class SigmaRenderCTI(RenderCTI):
15+
details: PlatformDetails = sigma_rule_details
16+
default_mapping = DEFAULT_SIGMA_CTI_MAPPING
17+
18+
def render(self, data: list[list[IocsChunkValue]]) -> list[str]:
19+
final_result = []
20+
for iocs_chunk in data:
21+
data_values = self.collect_sigma_data_values(iocs_chunk)
22+
rule = {
23+
"title": "Sigma automatically generated based on IOCs",
24+
"id": uuid.uuid4().__str__(),
25+
"description": "Detects suspicious activity based on IOCs.",
26+
"status": "experimental",
27+
"author": "SOC Prime",
28+
"logsource": {"product": "windows"},
29+
"fields": list(data_values.keys()),
30+
"detection": {"selection": data_values, "condition": "selection"},
31+
"level": SeverityType.low,
32+
"falsepositives": "",
33+
}
34+
final_result.append(yaml.dump(rule, default_flow_style=False, sort_keys=False))
35+
return final_result
36+
37+
def collect_sigma_data_values(self, chunk: list[IocsChunkValue]) -> dict:
38+
raw_data_values = {}
39+
for value in chunk:
40+
if value.platform_field in raw_data_values.keys():
41+
raw_data_values[value.platform_field].append(value.value)
42+
else:
43+
raw_data_values[value.platform_field] = [value.value]
44+
return raw_data_values
45+
46+
def generate(self, data: dict[list[list[IocsChunkValue]]], **kwargs):
47+
final_result = []
48+
for key, iocs_chunks in data.items():
49+
for iocs_chunk in iocs_chunks:
50+
data_values = self.collect_sigma_data_values(iocs_chunk)
51+
rule = {
52+
"title": f"IOCs ({key}) to detect: {kwargs['title']}",
53+
"id": uuid.uuid4().__str__(),
54+
"description": kwargs["description"],
55+
"status": "stable",
56+
"author": "SOC Prime Team",
57+
"logsource": LOGSOURCE_MAP.get(key),
58+
"fields": list(data_values.keys()),
59+
"detection": {"selection": data_values, "condition": "selection"},
60+
"level": SeverityType.medium,
61+
"falsepositives": "",
62+
"references": kwargs["references"],
63+
"date": kwargs["created_date"],
64+
"modified": kwargs["created_date"],
65+
}
66+
if kwargs.get("mitre_tags"):
67+
rule["tags"] = kwargs["mitre_tags"]
68+
final_result.append(yaml.dump(rule, default_flow_style=False, sort_keys=False))
69+
return final_result

‎uncoder-core/app/translator/tools/const.py

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,35 @@
1-
from typing import Literal
1+
import typing
22

33
IP_IOC_REGEXP_PATTERN = r"(?:^|[ \/\[(\"',;>|])((?:25[0-5]|2[0-4]\d|[0-1]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d{1,2})){3})(?=[\s)\]\"',;:\/?\n<|]|$)" # noqa: E501
44
DOMAIN_IOC_REGEXP_PATTERN = r"(?:^|[\s\/\[\]@(\"',;{>|])(?:(?:http[s]?|ftp):\/\/?)?([^:\\\/\s({\[\]@\"'`,]+\.[a-zA-Z]+)(?:(?:(?:[/|:]\w+)*\/)(?:[\w\-.]+[^#?\s]+)?(?:[\w/\-&?=%.#]+(?:\(\))?)?)?(?=[\s)\]\"',;<|]|$)" # noqa: E501
55
URL_IOC_REGEXP_PATTERN = r"(?:^|[\s\/\[\]@(\"',;{>|])((?:(?:http[s]?|ftp):\/\/?)+(?:[^:\\\/\s({\[\]@\"'`,]+\.[a-zA-Z0-9]+)(?:(?:(?:[/|:]\w+)*\/)(?:[\w\-.]+[^#?\s<']+)?(?:[\w/\-&?=%.#]+(?:\(\))?)?)?)(?=[\s)\]\"',;<|]|$)" # noqa: E501
66

7-
IOCType = Literal["ip", "domain", "url", "hash"]
8-
HashType = Literal["md5", "sha1", "sha256", "sha512"]
9-
IocParsingRule = Literal["replace_dots", "remove_private_and_reserved_ips", "replace_hxxp"]
7+
IOCType = typing.Literal["ip", "domain", "url", "hash"]
8+
HashType = typing.Literal["md5", "sha1", "sha256", "sha512"]
9+
IocParsingRule = typing.Literal["replace_dots", "remove_private_and_reserved_ips", "replace_hxxp"]
10+
11+
DefaultIOCType = list(typing.get_args(IOCType))
12+
DefaultHashType = list(typing.get_args(HashType))
13+
DefaultIocParsingRule = list(typing.get_args(IocParsingRule))
1014

1115
HASH_MAP = {"md5": "HashMd5", "sha1": "HashSha1", "sha256": "HashSha256", "sha512": "HashSha512"}
1216

17+
iocs_types_map = {
18+
"url": ["URL"],
19+
"domain": ["Domain"],
20+
"ip": ["DestinationIP", "SourceIP"],
21+
"hash": ["HashMd5", "HashSha1", "HashSha256", "HashSha512"],
22+
}
23+
24+
LOGSOURCE_MAP = {
25+
"hash": {"category": "process_creation"},
26+
"domain": {"category": "proxy"},
27+
"url": {"category": "proxy"},
28+
"ip": {"category": "proxy"},
29+
"emails": {"category": "mail"},
30+
"files": {"category": "file_event"},
31+
}
32+
1333
hash_regexes = {
1434
"md5": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{32})(?=[\s)\]\"',;\n<|]|$)",
1535
"sha1": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{40})(?=[\s)\]\"',;\n<|]|$)",

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy