Skip to content

Commit d69da7f

Browse files
nazargesyknazargesyk
committed
Merge branch 'main' into gis-improve_qradar_palo_alto_mapping
# Conflicts: # uncoder-core/app/translator/mappings/platforms/qradar/default.yml # uncoder-core/app/translator/mappings/platforms/qradar/linux_process_creation.yml # uncoder-core/app/translator/mappings/platforms/qradar/windows_process_creation.yml # uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml
2 parents fc38d67 + bf008fe commit d69da7f

File tree

229 files changed

+6655
-1653
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

229 files changed

+6655
-1653
lines changed

‎uncoder-core/app/translator/const.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@
99

1010
CTI_IOCS_PER_QUERY_LIMIT = 25
1111

12-
DEFAULT_VALUE_TYPE = Union[int, str, StrValue, list[Union[int, str, StrValue]]]
12+
DEFAULT_VALUE_TYPE = Union[bool, int, str, StrValue, list[Union[int, str, StrValue]]]

‎uncoder-core/app/translator/core/const.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from typing import Union
2+
3+
from app.translator.core.models.query_tokens.field import Alias, Field
4+
from app.translator.core.models.query_tokens.field_field import FieldField
5+
from app.translator.core.models.query_tokens.field_value import FieldValue
6+
from app.translator.core.models.query_tokens.function_value import FunctionValue
7+
from app.translator.core.models.query_tokens.identifier import Identifier
8+
from app.translator.core.models.query_tokens.keyword import Keyword
9+
10+
QUERY_TOKEN_TYPE = Union[FieldField, FieldValue, FunctionValue, Keyword, Identifier, Field, Alias]

‎uncoder-core/app/translator/core/context_vars.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
11
from contextvars import ContextVar
2+
from typing import Optional
23

34
return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False)
45
"""Set to True to return only first query if rendered multiple options"""
6+
7+
wrap_query_with_meta_info_ctx_var: ContextVar[bool] = ContextVar("wrap_query_with_meta_info_ctx_var", default=True)
8+
"""Set to False not to wrap query with meta info commentary"""
9+
10+
preset_log_source_str_ctx_var: ContextVar[Optional[str]] = ContextVar("preset_log_source_str_ctx_var", default=None)

‎uncoder-core/app/translator/core/custom_types/functions.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,21 @@ class FunctionType(CustomEnum):
1515
latest = "latest"
1616

1717
divide = "divide"
18+
multiply = "multiply"
1819

1920
lower = "lower"
2021
split = "split"
2122
upper = "upper"
2223

2324
array_length = "array_length"
24-
compare = "compare"
2525
extract_time = "extract_time"
2626
ipv4_is_in_range = "ipv4_is_in_range"
2727

2828
bin = "bin"
2929
eval = "eval"
3030
fields = "fields"
31+
iploc = "iploc"
32+
join = "join"
3133
rename = "rename"
3234
search = "search"
3335
sort_limit = "sort_limit"

‎uncoder-core/app/translator/core/custom_types/predefined_fields.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
from app.translator.tools.custom_enum import CustomEnum
2+
3+
4+
class IPLocationType(CustomEnum):
5+
asn = "ip_loc_asn"
6+
asn_org = "ip_loc_asn_org"
7+
city = "ip_loc_city"
8+
continent = "ip_loc_continent"
9+
country = "ip_loc_country"
10+
lat_lon = "ip_loc_lat_lon"
11+
region = "ip_loc_region"
12+
timezone = "ip_loc_timezone"
13+
14+
15+
class TimeType(CustomEnum):
16+
timestamp = "timestamp"

‎uncoder-core/app/translator/core/exceptions/core.py

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,25 @@
11
from typing import Optional
22

33

4-
class NotImplementedException(BaseException):
5-
...
6-
7-
84
class BasePlatformException(BaseException):
95
...
106

117

128
class StrictPlatformException(BasePlatformException):
13-
field_name: str = None
14-
15-
def __init__(
16-
self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None
17-
):
9+
def __init__(self, platform_name: str, fields: list[str], mapping: Optional[str] = None):
1810
message = (
1911
f"Platform {platform_name} has strict mapping. "
20-
f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping."
12+
f"Source fields: {', '.join(fields)} have no mapping."
2113
f" Mapping file: {mapping}."
2214
if mapping
2315
else ""
2416
)
25-
self.field_name = field_name
17+
super().__init__(message)
18+
19+
20+
class UnsupportedMappingsException(BasePlatformException):
21+
def __init__(self, platform_name: str, mappings: list[str]):
22+
message = f"Platform {platform_name} does not support these mappings: {mappings}."
2623
super().__init__(message)
2724

2825

@@ -93,5 +90,9 @@ class InvalidJSONStructure(InvalidRuleStructure):
9390
rule_type: str = "JSON"
9491

9592

93+
class InvalidTOMLStructure(InvalidRuleStructure):
94+
rule_type: str = "TOML"
95+
96+
9697
class InvalidXMLStructure(InvalidRuleStructure):
9798
rule_type: str = "XML"

‎uncoder-core/app/translator/core/exceptions/render.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,5 @@ class FunctionRenderException(BaseRenderException):
1414

1515
class UnsupportedRenderMethod(BaseRenderException):
1616
def __init__(self, platform_name: str, method: str):
17-
message = f"Cannot translate. {platform_name} backend does not support {method}."
17+
message = f'Cannot translate. {platform_name} backend does not support "{method}".'
1818
super().__init__(message)

‎uncoder-core/app/translator/core/functions.py

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525

2626
from app.translator.core.exceptions.functions import NotSupportedFunctionException
2727
from app.translator.core.mapping import SourceMapping
28-
from app.translator.core.models.field import Alias, Field
2928
from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions
29+
from app.translator.core.models.query_tokens.field import Alias, Field, PredefinedField
3030
from app.translator.tools.utils import execute_module
3131
from settings import INIT_FUNCTIONS
3232

@@ -83,7 +83,6 @@ def parse(self, func_body: str, raw: str) -> Function:
8383
class FunctionRender(ABC):
8484
function_names_map: ClassVar[dict[str, str]] = {}
8585
order_to_render: int = 0
86-
in_query_render: bool = False
8786
render_to_prefix: bool = False
8887
manager: PlatformFunctionsManager = None
8988

@@ -95,17 +94,19 @@ def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRe
9594
def render(self, function: Function, source_mapping: SourceMapping) -> str:
9695
raise NotImplementedError
9796

98-
@staticmethod
99-
def map_field(field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
97+
def map_field(self, field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
10098
if isinstance(field, Alias):
10199
return field.name
102100

103-
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
104-
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
105-
if isinstance(mapped_field, list):
106-
mapped_field = mapped_field[0]
101+
if isinstance(field, Field):
102+
mappings = self.manager.platform_functions.platform_query_render.mappings
103+
mapped_fields = mappings.map_field(field, source_mapping)
104+
return mapped_fields[0]
107105

108-
return mapped_field if mapped_field else field.source_name
106+
if isinstance(field, PredefinedField):
107+
return self.manager.platform_functions.platform_query_render.map_predefined_field(field)
108+
109+
raise NotSupportedFunctionException
109110

110111

111112
class PlatformFunctionsManager:
@@ -117,7 +118,6 @@ def __init__(self):
117118
self._parsers_map: dict[str, FunctionParser] = {} # {platform_func_name: FunctionParser}
118119

119120
self._renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
120-
self._in_query_renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
121121
self._order_to_render: dict[str, int] = {} # {generic_func_name: int}
122122

123123
def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRender]:
@@ -126,8 +126,6 @@ def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRe
126126
for generic_function_name in render.function_names_map:
127127
self._renders_map[generic_function_name] = render
128128
self._order_to_render[generic_function_name] = render.order_to_render
129-
if render.in_query_render:
130-
self._in_query_renders_map[generic_function_name] = render
131129

132130
return render_class
133131

@@ -149,24 +147,16 @@ def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser:
149147

150148
raise NotSupportedFunctionException
151149

152-
def get_parser(self, platform_func_name: str) -> FunctionParser:
150+
def get_parser(self, platform_func_name: str) -> Optional[FunctionParser]:
153151
if INIT_FUNCTIONS and (parser := self._parsers_map.get(platform_func_name)):
154152
return parser
155153

156-
raise NotSupportedFunctionException
157-
158154
def get_render(self, generic_func_name: str) -> FunctionRender:
159155
if INIT_FUNCTIONS and (render := self._renders_map.get(generic_func_name)):
160156
return render
161157

162158
raise NotSupportedFunctionException
163159

164-
def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
165-
if INIT_FUNCTIONS and (render := self._in_query_renders_map.get(generic_func_name)):
166-
return render
167-
168-
raise NotSupportedFunctionException
169-
170160
@property
171161
def order_to_render(self) -> dict[str, int]:
172162
if INIT_FUNCTIONS:

‎uncoder-core/app/translator/core/mapping.py

Lines changed: 88 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
from __future__ import annotations
22

33
from abc import ABC, abstractmethod
4-
from typing import Optional, TypeVar
4+
from typing import TYPE_CHECKING, Optional, TypeVar, Union
55

6+
from app.translator.core.exceptions.core import StrictPlatformException, UnsupportedMappingsException
7+
from app.translator.core.models.platform_details import PlatformDetails
68
from app.translator.mappings.utils.load_from_files import LoaderFileMappings
79

10+
if TYPE_CHECKING:
11+
from app.translator.core.models.query_tokens.field import Field
12+
13+
814
DEFAULT_MAPPING_NAME = "default"
915

1016

@@ -13,9 +19,14 @@ class LogSourceSignature(ABC):
1319
wildcard_symbol = "*"
1420

1521
@abstractmethod
16-
def is_suitable(self, *args, **kwargs) -> bool:
22+
def is_suitable(self, **kwargs) -> bool:
1723
raise NotImplementedError("Abstract method")
1824

25+
@staticmethod
26+
def _check_conditions(conditions: list[Union[bool, None]]) -> bool:
27+
conditions = [condition for condition in conditions if condition is not None]
28+
return bool(conditions) and all(conditions)
29+
1930
@abstractmethod
2031
def __str__(self) -> str:
2132
raise NotImplementedError("Abstract method")
@@ -64,7 +75,7 @@ def update(self, fields_mapping: FieldsMapping) -> None:
6475
self.__render_mapping.update(fields_mapping.__render_mapping)
6576

6677
def is_suitable(self, field_names: list[str]) -> bool:
67-
return set(field_names).issubset(set(self.__parser_mapping.keys()))
78+
return bool(field_names) and set(field_names).issubset(set(self.__parser_mapping.keys()))
6879

6980

7081
_LogSourceSignatureType = TypeVar("_LogSourceSignatureType", bound=LogSourceSignature)
@@ -85,12 +96,16 @@ def __init__(
8596

8697

8798
class BasePlatformMappings:
99+
details: PlatformDetails = None
100+
101+
is_strict_mapping: bool = False
88102
skip_load_default_mappings: bool = True
89103
extend_default_mapping_with_all_fields: bool = False
90104

91-
def __init__(self, platform_dir: str):
105+
def __init__(self, platform_dir: str, platform_details: PlatformDetails):
92106
self._loader = LoaderFileMappings()
93107
self._platform_dir = platform_dir
108+
self.details = platform_details
94109
self._source_mappings = self.prepare_mapping()
95110

96111
def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None:
@@ -137,9 +152,34 @@ def prepare_fields_mapping(field_mapping: dict) -> FieldsMapping:
137152
def prepare_log_source_signature(self, mapping: dict) -> LogSourceSignature:
138153
raise NotImplementedError("Abstract method")
139154

140-
@abstractmethod
141-
def get_suitable_source_mappings(self, *args, **kwargs) -> list[SourceMapping]:
142-
raise NotImplementedError("Abstract method")
155+
def get_source_mappings_by_fields_and_log_sources(
156+
self, field_names: list[str], log_sources: dict[str, list[Union[int, str]]]
157+
) -> list[SourceMapping]:
158+
by_log_sources_and_fields = []
159+
by_fields = []
160+
for source_mapping in self._source_mappings.values():
161+
if source_mapping.source_id == DEFAULT_MAPPING_NAME:
162+
continue
163+
164+
if source_mapping.fields_mapping.is_suitable(field_names):
165+
by_fields.append(source_mapping)
166+
167+
log_source_signature: LogSourceSignature = source_mapping.log_source_signature
168+
if log_source_signature and log_source_signature.is_suitable(**log_sources):
169+
by_log_sources_and_fields.append(source_mapping)
170+
171+
return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]]
172+
173+
def get_source_mappings_by_ids(self, source_mapping_ids: list[str]) -> list[SourceMapping]:
174+
source_mappings = []
175+
for source_mapping_id in source_mapping_ids:
176+
if source_mapping := self.get_source_mapping(source_mapping_id):
177+
source_mappings.append(source_mapping)
178+
179+
if not source_mappings:
180+
source_mappings = [self.get_source_mapping(DEFAULT_MAPPING_NAME)]
181+
182+
return source_mappings
143183

144184
def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
145185
return self._source_mappings.get(source_id)
@@ -148,6 +188,32 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
148188
def default_mapping(self) -> SourceMapping:
149189
return self._source_mappings[DEFAULT_MAPPING_NAME]
150190

191+
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
192+
unmapped = []
193+
for field in field_tokens:
194+
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
195+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
196+
if not mapped_field and field.source_name not in unmapped:
197+
unmapped.append(field.source_name)
198+
199+
if self.is_strict_mapping and unmapped:
200+
raise StrictPlatformException(
201+
platform_name=self.details.name, fields=unmapped, mapping=source_mapping.source_id
202+
)
203+
204+
return unmapped
205+
206+
@staticmethod
207+
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
208+
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
209+
# field can be mapped to corresponding platform field name or list of platform field names
210+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
211+
212+
if isinstance(mapped_field, str):
213+
mapped_field = [mapped_field]
214+
215+
return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]
216+
151217

152218
class BaseCommonPlatformMappings(ABC, BasePlatformMappings):
153219
def prepare_mapping(self) -> dict[str, SourceMapping]:
@@ -163,3 +229,18 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
163229
)
164230

165231
return source_mappings
232+
233+
234+
class BaseStrictLogSourcesPlatformMappings(ABC, BasePlatformMappings):
235+
def get_source_mappings_by_ids(self, source_mapping_ids: list[str]) -> list[SourceMapping]:
236+
source_mappings = []
237+
for source_mapping_id in source_mapping_ids:
238+
if source_mapping_id == DEFAULT_MAPPING_NAME:
239+
continue
240+
if source_mapping := self.get_source_mapping(source_mapping_id):
241+
source_mappings.append(source_mapping)
242+
243+
if not source_mappings:
244+
raise UnsupportedMappingsException(platform_name=self.details.name, mappings=source_mapping_ids)
245+
246+
return source_mappings

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy