Skip to content

Commit 70f1371

Browse files
nazargesyknazargesyk
authored
Merge branch 'main' into gis-8882
2 parents fd35d58 + 0e5e0ca commit 70f1371

File tree

104 files changed

+944
-301
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

104 files changed

+944
-301
lines changed

‎uncoder-core/app/routers/meta_info.py

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
from dataclasses import asdict
2+
3+
from fastapi import APIRouter, Body, HTTPException
4+
5+
from app.models.meta_info import (
6+
MetaInfo,
7+
MetaInfoResponse,
8+
MitreInfoContainer,
9+
MitreTacticContainer,
10+
MitreTechniqueContainer,
11+
ParsedLogSources,
12+
RawMetaInfo,
13+
)
14+
from app.translator.core.exceptions.core import UnsupportedPlatform
15+
from app.translator.translator import app_translator
16+
17+
meta_info_router = APIRouter()
18+
19+
20+
@meta_info_router.post("/get_meta_info/", tags=["meta_info"], description="Get Rule MetaInfo")
21+
@meta_info_router.post("/get_meta_info/", include_in_schema=False)
22+
def get_meta_info_data(
23+
source_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True)
24+
) -> MetaInfoResponse:
25+
try:
26+
logsources, raw_query_container = app_translator.parse_meta_info(text=text, source=source_platform_id)
27+
except UnsupportedPlatform as exc:
28+
raise HTTPException(status_code=400, detail="Unsuported platform") from exc
29+
except Exception as exc:
30+
raise HTTPException(status_code=400, detail="Unexpected error.") from exc
31+
if not raw_query_container:
32+
raise HTTPException(status_code=400, detail="Can't parse metadata")
33+
most_frequent_product = max(logsources.get("product"), key=logsources.get("product").get, default=None)
34+
most_frequent_service = max(logsources.get("service"), key=logsources.get("service").get, default=None)
35+
most_frequent_category = max(logsources.get("category"), key=logsources.get("category").get, default=None)
36+
37+
logsources.get("product", {}).pop(most_frequent_product, None)
38+
logsources.get("service", {}).pop(most_frequent_service, None)
39+
logsources.get("category", {}).pop(most_frequent_category, None)
40+
41+
parsed_logsources = ParsedLogSources(
42+
most_frequent_product=most_frequent_product,
43+
most_frequent_service=most_frequent_service,
44+
most_frequent_category=most_frequent_category,
45+
least_frequent_products=list(logsources.get("product", {}).keys()),
46+
least_frequent_services=list(logsources.get("service", {}).keys()),
47+
least_frequent_categories=list(logsources.get("category", {}).keys()),
48+
)
49+
return MetaInfoResponse(
50+
query=raw_query_container.query,
51+
language=raw_query_container.language,
52+
meta_info=MetaInfo(
53+
id_=raw_query_container.meta_info.id,
54+
title=raw_query_container.meta_info.title,
55+
description=raw_query_container.meta_info.description,
56+
author=raw_query_container.meta_info.author,
57+
date=raw_query_container.meta_info.date,
58+
false_positives=raw_query_container.meta_info.false_positives,
59+
license_=raw_query_container.meta_info.license,
60+
mitre_attack=MitreInfoContainer(
61+
tactics=[
62+
MitreTacticContainer(**asdict(tactic_container))
63+
for tactic_container in raw_query_container.meta_info.mitre_attack.tactics
64+
],
65+
techniques=[
66+
MitreTechniqueContainer(**asdict(tactic_container))
67+
for tactic_container in raw_query_container.meta_info.mitre_attack.techniques
68+
],
69+
),
70+
output_table_fields=raw_query_container.meta_info.output_table_fields,
71+
parsed_log_sources=parsed_logsources,
72+
query_fields=raw_query_container.meta_info.query_fields,
73+
query_period=raw_query_container.meta_info.query_period,
74+
raw_metainfo_container=RawMetaInfo(
75+
trigger_operator=raw_query_container.meta_info.raw_metainfo_container.trigger_operator,
76+
trigger_threshold=raw_query_container.meta_info.raw_metainfo_container.trigger_threshold,
77+
query_frequency=raw_query_container.meta_info.raw_metainfo_container.query_frequency,
78+
query_period=raw_query_container.meta_info.raw_metainfo_container.query_period,
79+
),
80+
raw_mitre_attack=raw_query_container.meta_info.raw_mitre_attack,
81+
references=raw_query_container.meta_info.references,
82+
severity=raw_query_container.meta_info.severity,
83+
source_mapping_ids=raw_query_container.meta_info.source_mapping_ids,
84+
status=raw_query_container.meta_info.status,
85+
tags=raw_query_container.meta_info.tags,
86+
timeframe=raw_query_container.meta_info.timeframe,
87+
),
88+
)

‎uncoder-core/app/translator/core/functions.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,10 @@ def order_to_render(self) -> dict[str, int]:
164164

165165
return {}
166166

167+
@property
168+
def supported_render_names(self) -> set[str]:
169+
return set(self._renders_map)
170+
167171

168172
class PlatformFunctions:
169173
dir_path: str = None

‎uncoder-core/app/translator/core/mapping.py

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ class LogSourceSignature(ABC):
2222
def is_suitable(self, **kwargs) -> bool:
2323
raise NotImplementedError("Abstract method")
2424

25+
def is_probably_suitable(self, **kwargs) -> bool:
26+
"""
27+
Performs check with more options, but the result is less accurate than the "is_suitable" method
28+
"""
29+
raise NotImplementedError("Abstract method")
30+
2531
@staticmethod
2632
def _check_conditions(conditions: list[Union[bool, None]]) -> bool:
2733
conditions = [condition for condition in conditions if condition is not None]
@@ -88,11 +94,13 @@ def __init__(
8894
log_source_signature: _LogSourceSignatureType = None,
8995
fields_mapping: Optional[FieldsMapping] = None,
9096
raw_log_fields: Optional[dict] = None,
97+
conditions: Optional[dict] = None,
9198
):
9299
self.source_id = source_id
93100
self.log_source_signature = log_source_signature
94101
self.fields_mapping = fields_mapping or FieldsMapping([])
95102
self.raw_log_fields = raw_log_fields
103+
self.conditions = conditions
96104

97105

98106
class BasePlatformMappings:
@@ -123,6 +131,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
123131

124132
field_mappings_dict = mapping_dict.get("field_mapping", {})
125133
raw_log_fields = mapping_dict.get("raw_log_fields", {})
134+
conditions = mapping_dict.get("conditions", {})
126135
field_mappings_dict.update({field: field for field in raw_log_fields})
127136
fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict)
128137
self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping)
@@ -131,6 +140,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
131140
log_source_signature=log_source_signature,
132141
fields_mapping=fields_mapping,
133142
raw_log_fields=raw_log_fields,
143+
conditions=conditions,
134144
)
135145

136146
if self.skip_load_default_mappings:
@@ -170,31 +180,47 @@ def get_source_mappings_by_fields_and_log_sources(
170180

171181
return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]]
172182

173-
def get_source_mappings_by_ids(self, source_mapping_ids: list[str]) -> list[SourceMapping]:
183+
def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
184+
return self._source_mappings.get(source_id)
185+
186+
def get_source_mappings_by_ids(
187+
self, source_mapping_ids: list[str], return_default: bool = True
188+
) -> list[SourceMapping]:
174189
source_mappings = []
175190
for source_mapping_id in source_mapping_ids:
191+
if source_mapping_id == DEFAULT_MAPPING_NAME:
192+
continue
176193
if source_mapping := self.get_source_mapping(source_mapping_id):
177194
source_mappings.append(source_mapping)
178195

179-
if not source_mappings:
196+
if not source_mappings and return_default:
180197
source_mappings = [self.get_source_mapping(DEFAULT_MAPPING_NAME)]
181198

182199
return source_mappings
183200

184-
def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
185-
return self._source_mappings.get(source_id)
201+
def get_source_mappings_by_log_sources(self, log_sources: dict) -> Optional[list[str]]:
202+
raise NotImplementedError("Abstract method")
186203

187204
@property
188205
def default_mapping(self) -> SourceMapping:
189206
return self._source_mappings[DEFAULT_MAPPING_NAME]
190207

191-
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
208+
def check_fields_mapping_existence(
209+
self,
210+
query_field_tokens: list[Field],
211+
function_field_tokens_map: dict[str, list[Field]],
212+
supported_func_render_names: set[str],
213+
source_mapping: SourceMapping,
214+
) -> list[str]:
192215
unmapped = []
193-
for field in field_tokens:
194-
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
195-
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
196-
if not mapped_field and field.source_name not in unmapped:
197-
unmapped.append(field.source_name)
216+
217+
for field in query_field_tokens:
218+
self._check_field_mapping_existence(field, source_mapping, unmapped)
219+
220+
for func_name, function_field_tokens in function_field_tokens_map.items():
221+
if func_name in supported_func_render_names:
222+
for field in function_field_tokens:
223+
self._check_field_mapping_existence(field, source_mapping, unmapped)
198224

199225
if self.is_strict_mapping and unmapped:
200226
raise StrictPlatformException(
@@ -203,6 +229,13 @@ def check_fields_mapping_existence(self, field_tokens: list[Field], source_mappi
203229

204230
return unmapped
205231

232+
@staticmethod
233+
def _check_field_mapping_existence(field: Field, source_mapping: SourceMapping, unmapped: list[str]) -> None:
234+
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
235+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
236+
if not mapped_field and field.source_name not in unmapped:
237+
unmapped.append(field.source_name)
238+
206239
@staticmethod
207240
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
208241
generic_field_name = field.get_generic_field_name(source_mapping.source_id)

‎uncoder-core/app/translator/core/mixins/rule.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ def parse_mitre_attack(self, tags: list[str]) -> MitreInfoContainer:
4242
tag = tag.lower()
4343
if tag.startswith("attack."):
4444
tag = tag[7::]
45-
if tag.startswith("t"):
45+
if tag[-1].isdigit():
4646
parsed_techniques.append(tag)
4747
else:
4848
parsed_tactics.append(tag)

‎uncoder-core/app/translator/core/models/query_container.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ def __init__(
6565
date: Optional[str] = None,
6666
output_table_fields: Optional[list[Field]] = None,
6767
query_fields: Optional[list[Field]] = None,
68+
function_fields: Optional[list[Field]] = None,
69+
function_fields_map: Optional[dict[str, list[Field]]] = None,
6870
license_: Optional[str] = None,
6971
severity: Optional[str] = None,
7072
references: Optional[list[str]] = None,
@@ -76,7 +78,7 @@ def __init__(
7678
parsed_logsources: Optional[dict] = None,
7779
timeframe: Optional[timedelta] = None,
7880
query_period: Optional[timedelta] = None,
79-
mitre_attack: MitreInfoContainer = MitreInfoContainer(),
81+
mitre_attack: Optional[MitreInfoContainer] = None,
8082
raw_metainfo_container: Optional[RawMetaInfoContainer] = None,
8183
) -> None:
8284
self.id = id_ or str(uuid.uuid4())
@@ -86,23 +88,25 @@ def __init__(
8688
self.risk_score = risk_score
8789
self.type_ = type_ or ""
8890
self.description = description or ""
89-
self.author = [v.strip() for v in author] if author else []
91+
self.author = [v.strip() for v in author] if author and author != [None] else []
9092
self.date = date or datetime.now().date().strftime("%Y-%m-%d")
9193
self.output_table_fields = output_table_fields or []
9294
self.query_fields = query_fields or []
95+
self.function_fields = function_fields or []
96+
self.function_fields_map = function_fields_map or {}
9397
self.license = license_ or "DRL 1.1"
9498
self.severity = severity or SeverityType.low
9599
self.references = references or []
96100
self.tags = tags or []
97-
self.mitre_attack = mitre_attack or None
101+
self.mitre_attack = mitre_attack or MitreInfoContainer()
98102
self.raw_mitre_attack = raw_mitre_attack or []
99103
self.status = status or "stable"
100104
self.false_positives = false_positives or []
101105
self._source_mapping_ids = source_mapping_ids or [DEFAULT_MAPPING_NAME]
102106
self.parsed_logsources = parsed_logsources or {}
103107
self.timeframe = timeframe
104108
self.query_period = query_period
105-
self.raw_metainfo_container = raw_metainfo_container
109+
self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer()
106110

107111
@property
108112
def author_str(self) -> str:

‎uncoder-core/app/translator/core/parser.py

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,16 +65,19 @@ def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]:
6565
@staticmethod
6666
def get_field_tokens(
6767
query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None
68-
) -> list[Field]:
69-
field_tokens = []
68+
) -> tuple[list[Field], list[Field], dict[str, list[Field]]]:
69+
query_field_tokens = []
70+
function_field_tokens = []
71+
function_field_tokens_map = {}
7072
for token in query_tokens:
7173
if isinstance(token, (FieldField, FieldValue, FunctionValue)):
72-
field_tokens.extend(token.fields)
74+
query_field_tokens.extend(token.fields)
7375

74-
if functions:
75-
field_tokens.extend([field for func in functions for field in func.fields])
76+
for func in functions or []:
77+
function_field_tokens.extend(func.fields)
78+
function_field_tokens_map[func.name] = func.fields
7679

77-
return field_tokens
80+
return query_field_tokens, function_field_tokens, function_field_tokens_map
7881

7982
def get_source_mappings(
8083
self, field_tokens: list[Field], log_sources: dict[str, list[Union[int, str]]]

‎uncoder-core/app/translator/core/render.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -428,14 +428,18 @@ def _generate_from_tokenized_query_container_by_source_mapping(
428428
self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping
429429
) -> str:
430430
unmapped_fields = self.mappings.check_fields_mapping_existence(
431-
query_container.meta_info.query_fields, source_mapping
431+
query_container.meta_info.query_fields,
432+
query_container.meta_info.function_fields_map,
433+
self.platform_functions.manager.supported_render_names,
434+
source_mapping,
432435
)
433436
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
434437
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)
435438

436439
if source_mapping.raw_log_fields:
437440
defined_raw_log_fields = self.generate_raw_log_fields(
438-
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
441+
fields=query_container.meta_info.query_fields + query_container.meta_info.function_fields,
442+
source_mapping=source_mapping,
439443
)
440444
prefix += f"\n{defined_raw_log_fields}"
441445
query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)

‎uncoder-core/app/translator/core/str_value_manager.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,11 @@ class StrValueManager:
182182
container_spec_symbols_map: ClassVar[dict[type[BaseSpecSymbol], str]] = CONTAINER_SPEC_SYMBOLS_MAP
183183

184184
@staticmethod
185-
def from_str_to_container(value: str) -> StrValue:
185+
def from_str_to_container(
186+
value: str,
187+
value_type: str = ValueType.value, # noqa: ARG004
188+
escape_symbol: Optional[str] = None, # noqa: ARG004
189+
) -> StrValue:
186190
return StrValue(value=value, split_value=[value])
187191

188192
def from_re_str_to_container(self, value: str) -> StrValue:

‎uncoder-core/app/translator/core/tokenizer.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,8 +162,7 @@ def search_multi_value(
162162

163163
def _get_field_value_match(self, query: str, operator: str, field_name: str, value_pattern: str) -> re.Match:
164164
field_value_pattern = self.get_field_value_pattern(operator, field_name, value_pattern)
165-
field_value_regex = re.compile(field_value_pattern, re.IGNORECASE)
166-
field_value_match = re.match(field_value_regex, query)
165+
field_value_match = re.match(field_value_pattern, query, re.IGNORECASE)
167166
if field_value_match is None:
168167
raise TokenizerGeneralException(error=f"Value couldn't be found in query part: {query}")
169168

‎uncoder-core/app/translator/mappings/platforms/anomali/common.yml renamed to ‎uncoder-core/app/translator/mappings/platforms/anomali/proxy.yml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
11
platform: Anomali
2-
description: Common field mapping
2+
source: proxy
33

44
field_mapping:
55
c-uri-query: url
66
c-useragent: user_agent
7+
c-uri: url
8+
cs-method: http_method
9+
cs-bytes: bytes_out
10+
cs-referrer: http_referrer
11+
sc-status: return_code
12+
13+
dns-query: query
14+
dns-answer: answer
15+
dns-record: record_type
16+
717
CommandLine: command_line
818
DestinationHostname: dest
919
DestinationIp: dest_ip

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy