Skip to content

Commit 40f8c29

Browse files
alexvolhaalexvolha
committed
resolve conflicts
1 parent c381cf6 commit 40f8c29

File tree

34 files changed

+504
-72
lines changed

34 files changed

+504
-72
lines changed

‎uncoder-core/app/translator/core/mapping.py

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -72,34 +72,49 @@ def __init__(
7272
source_id: str,
7373
log_source_signature: _LogSourceSignatureType = None,
7474
fields_mapping: Optional[FieldsMapping] = None,
75+
raw_log_fields: Optional[list] = None,
7576
):
7677
self.source_id = source_id
7778
self.log_source_signature = log_source_signature
7879
self.fields_mapping = fields_mapping or FieldsMapping([])
80+
self.raw_log_fields = raw_log_fields
7981

8082

8183
class BasePlatformMappings:
84+
skip_load_default_mappings: bool = True
85+
8286
def __init__(self, platform_dir: str):
8387
self._loader = LoaderFileMappings()
8488
self._platform_dir = platform_dir
8589
self._source_mappings = self.prepare_mapping()
8690

91+
def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None:
92+
default_mapping.fields_mapping.update(fields_mapping)
93+
8794
def prepare_mapping(self) -> dict[str, SourceMapping]:
8895
source_mappings = {}
8996
default_mapping = SourceMapping(source_id=DEFAULT_MAPPING_NAME)
9097
for mapping_dict in self._loader.load_platform_mappings(self._platform_dir):
9198
log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict)
9299
if (source_id := mapping_dict["source"]) == DEFAULT_MAPPING_NAME:
93100
default_mapping.log_source_signature = log_source_signature
94-
continue
95-
96-
fields_mapping = self.prepare_fields_mapping(field_mapping=mapping_dict.get("field_mapping", {}))
97-
default_mapping.fields_mapping.update(fields_mapping)
101+
if self.skip_load_default_mappings:
102+
continue
103+
104+
field_mappings_dict = mapping_dict.get("field_mapping", {})
105+
raw_log_fields = mapping_dict.get("raw_log_fields", [])
106+
field_mappings_dict.update({field: field for field in raw_log_fields})
107+
fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict)
108+
self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping)
98109
source_mappings[source_id] = SourceMapping(
99-
source_id=source_id, log_source_signature=log_source_signature, fields_mapping=fields_mapping
110+
source_id=source_id,
111+
log_source_signature=log_source_signature,
112+
fields_mapping=fields_mapping,
113+
raw_log_fields=raw_log_fields,
100114
)
101115

102-
source_mappings[DEFAULT_MAPPING_NAME] = default_mapping
116+
if self.skip_load_default_mappings:
117+
source_mappings[DEFAULT_MAPPING_NAME] = default_mapping
103118

104119
return source_mappings
105120

‎uncoder-core/app/translator/core/models/query_container.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ def __init__(
1919
description: Optional[str] = None,
2020
author: Optional[str] = None,
2121
date: Optional[str] = None,
22-
fields: Optional[list[Field]] = None,
22+
output_table_fields: Optional[list[Field]] = None,
23+
query_fields: Optional[list[Field]] = None,
2324
license_: Optional[str] = None,
2425
severity: Optional[str] = None,
2526
references: Optional[list[str]] = None,
@@ -35,7 +36,8 @@ def __init__(
3536
self.description = description or ""
3637
self.author = author or ""
3738
self.date = date or datetime.now().date().strftime("%Y-%m-%d")
38-
self.fields = fields or []
39+
self.output_table_fields = output_table_fields or []
40+
self.query_fields = query_fields or []
3941
self.license = license_ or "DRL 1.1"
4042
self.severity = severity or SeverityType.low
4143
self.references = references or []

‎uncoder-core/app/translator/core/parser.py

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,27 @@
1515
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1616
-----------------------------------------------------------------
1717
"""
18-
18+
import re
1919
from abc import ABC, abstractmethod
2020
from typing import Union
2121

2222
from app.translator.core.exceptions.parser import TokenizerGeneralException
2323
from app.translator.core.functions import PlatformFunctions
2424
from app.translator.core.mapping import BasePlatformMappings, SourceMapping
25-
from app.translator.core.models.field import FieldValue
25+
from app.translator.core.models.field import Field, FieldValue, Keyword
2626
from app.translator.core.models.functions.base import ParsedFunctions
27+
from app.translator.core.models.identifier import Identifier
2728
from app.translator.core.models.platform_details import PlatformDetails
2829
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
2930
from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer
3031

3132

3233
class QueryParser(ABC):
34+
wrapped_with_comment_pattern: str = None
35+
36+
def remove_comments(self, text: str) -> str:
37+
return re.sub(self.wrapped_with_comment_pattern, "\n", text, flags=re.MULTILINE).strip()
38+
3339
def parse_raw_query(self, text: str, language: str) -> RawQueryContainer:
3440
return RawQueryContainer(query=text, language=language)
3541

@@ -44,13 +50,16 @@ class PlatformQueryParser(QueryParser, ABC):
4450
details: PlatformDetails = None
4551
platform_functions: PlatformFunctions = None
4652

53+
def get_fields_tokens(self, tokens: list[Union[FieldValue, Keyword, Identifier]]) -> list[Field]:
54+
return [token.field for token in self.tokenizer.filter_tokens(tokens, FieldValue)]
55+
4756
def get_tokens_and_source_mappings(
4857
self, query: str, log_sources: dict[str, Union[str, list[str]]]
4958
) -> tuple[list[TOKEN_TYPE], list[SourceMapping]]:
5059
if not query:
5160
raise TokenizerGeneralException("Can't translate empty query. Please provide more details")
5261
tokens = self.tokenizer.tokenize(query=query)
53-
field_tokens = [token.field for token in self.tokenizer.filter_tokens(tokens, FieldValue)]
62+
field_tokens = self.get_fields_tokens(tokens=tokens)
5463
field_names = [field.source_name for field in field_tokens]
5564
source_mappings = self.mappings.get_suitable_source_mappings(field_names=field_names, **log_sources)
5665
self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping)

‎uncoder-core/app/translator/core/render.py

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ def __init__(self, or_token: str):
5959
OperatorType.REGEX: self.regex_modifier,
6060
OperatorType.NOT_REGEX: self.not_regex_modifier,
6161
OperatorType.KEYWORD: self.keywords,
62+
OperatorType.IS_NONE: self.is_none,
63+
OperatorType.IS_NOT_NONE: self.is_not_none,
6264
}
6365
self.or_token = f" {or_token} "
6466

@@ -107,6 +109,12 @@ def not_regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # n
107109
def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002
108110
raise NotImplementedException
109111

112+
def is_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002
113+
raise NotImplementedException
114+
115+
def is_not_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG002
116+
raise NotImplementedException
117+
110118
def apply_value(self, value: Union[str, int], value_type: str = ValueType.value) -> Union[str, int]:
111119
return self.escape_manager.escape(value, value_type)
112120

@@ -118,13 +126,13 @@ def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VAL
118126

119127
class QueryRender(ABC):
120128
comment_symbol: str = None
121-
is_multi_line_comment: bool = False
129+
is_single_line_comment: bool = False
122130
unsupported_functions_text = "Unsupported functions were excluded from the result query:"
123131

124132
platform_functions: PlatformFunctions = PlatformFunctions()
125133

126134
def render_not_supported_functions(self, not_supported_functions: list) -> str:
127-
line_template = f"{self.comment_symbol} " if self.comment_symbol and self.is_multi_line_comment else ""
135+
line_template = f"{self.comment_symbol} " if self.comment_symbol and self.is_single_line_comment else ""
128136
not_supported_functions_str = "\n".join(line_template + func.lstrip() for func in not_supported_functions)
129137
return "\n\n" + self.wrap_with_comment(f"{self.unsupported_functions_text}\n{not_supported_functions_str}")
130138

@@ -139,7 +147,7 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta
139147
class PlatformQueryRender(QueryRender):
140148
mappings: BasePlatformMappings = None
141149
details: PlatformDetails = None
142-
is_strict_mapping = False
150+
is_strict_mapping: bool = False
143151

144152
or_token = "or"
145153
and_token = "and"
@@ -150,6 +158,7 @@ class PlatformQueryRender(QueryRender):
150158
field_value_map = BaseQueryFieldValue(or_token=or_token)
151159

152160
query_pattern = "{table} {query} {functions}"
161+
raw_log_field_pattern: str = None
153162

154163
def __init__(self):
155164
self.operator_map = {
@@ -272,12 +281,29 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer)
272281
prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info
273282
)
274283

284+
def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str:
285+
defined_raw_log_fields = []
286+
for field in fields:
287+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=field.source_name)
288+
if not mapped_field and self.is_strict_mapping:
289+
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
290+
if mapped_field not in source_mapping.raw_log_fields:
291+
continue
292+
field_prefix = self.raw_log_field_pattern.format(field=mapped_field)
293+
defined_raw_log_fields.append(field_prefix)
294+
return "\n".join(defined_raw_log_fields)
295+
275296
def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
276297
queries_map = {}
277298
source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids)
278299

279300
for source_mapping in source_mappings:
280301
prefix = self.generate_prefix(source_mapping.log_source_signature)
302+
if source_mapping.raw_log_fields:
303+
defined_raw_log_fields = self.generate_raw_log_fields(
304+
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
305+
)
306+
prefix += f"\n{defined_raw_log_fields}\n"
281307
result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
282308
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
283309
not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,18 @@ source: default
33

44

55
default_log_source:
6-
datamodel: datamodel
6+
datamodel: datamodel
7+
8+
9+
field_mapping:
10+
CommandLine: xdm.target.process.command_line
11+
Image:
12+
- xdm.target.process.name
13+
- xdm.source.process.name
14+
ParentCommandLine: xdm.source.process.command_line
15+
ParentImage: xdm.source.process.name
16+
User: xdm.source.user.username
17+
TargetFilename: xdm.target.file.filename
18+
TargetImage: xdm.target.process.name
19+
SourceImage: xdm.source.process.name
20+
EventID: action_evtlog_event_id

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/firewall.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,16 @@ field_mapping:
2020
SourceIp:
2121
- action_local_ip
2222
- action_remote_ip
23-
dst_ip:
23+
dst-ip:
2424
- action_local_ip
2525
- action_remote_ip
26-
dst_port:
26+
dst-port:
2727
- action_local_port
2828
- action_remote_port
29-
src_ip:
29+
src-ip:
3030
- action_local_ip
3131
- action_remote_ip
32-
src_port:
32+
src-port:
3333
- action_local_port
3434
- action_remote_port
3535
Protocol: action_network_protocol

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_application.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
platform: Palo Alto XSIAM
2+
source: windows_application
3+
4+
default_log_source:
5+
dataset: microsoft_windows_raw
6+
7+
field_mapping:
8+
EventID: action_evtlog_event_id
9+
10+
raw_log_fields:
11+
- src_ip
12+
- source
13+
- additional_information
14+
- EventData
15+
- Channel
16+
- statement
17+
- Faulting application path
18+
- object_name
19+
- class_type
20+
- action_id
21+
- Provider_Name
22+
- Data
23+
- Message
24+
- Level

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_powershell.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
platform: Palo Alto XSIAM
2+
source: windows_powershell
3+
4+
5+
default_log_source:
6+
dataset: microsoft_windows_raw
7+
8+
field_mapping:
9+
EventID: action_evtlog_event_id
10+
11+
12+
raw_log_fields:
13+
- CommandLine
14+
- ScriptBlockText
15+
- Payload
16+
- HostApplication
17+
- ContextInfo
18+
- HostName
19+
- EngineVersion

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy