Skip to content

Commit 5fd269c

Browse files
alexvolhaalexvolha
committed
predefined field class
1 parent c1dddc7 commit 5fd269c

File tree

8 files changed

+75
-23
lines changed

8 files changed

+75
-23
lines changed

‎uncoder-core/app/translator/core/custom_types/predefined_fields.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from app.translator.tools.custom_enum import CustomEnum
2+
3+
4+
class IPLocationType(CustomEnum):
5+
asn = "ip_loc_asn"
6+
asn_org = "ip_loc_asn_org"
7+
city = "ip_loc_city"
8+
continent = "ip_loc_continent"
9+
country = "ip_loc_country"
10+
lat_lon = "ip_loc_lat_lon"
11+
region = "ip_loc_region"
12+
timezone = "ip_loc_timezone"

‎uncoder-core/app/translator/core/models/field.py

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma
3737
self.__generic_names_map = generic_names_map
3838

3939

40+
class PredefinedField:
41+
def __init__(self, name: str):
42+
self.name = name
43+
44+
4045
class FieldField:
4146
def __init__(
4247
self,
@@ -46,10 +51,10 @@ def __init__(
4651
is_alias_left: bool = False,
4752
is_alias_right: bool = False,
4853
):
49-
self.field_left = Field(source_name=source_name_left)
54+
self.field_left = Field(source_name=source_name_left) if not is_alias_left else None
5055
self.alias_left = Alias(name=source_name_left) if is_alias_left else None
5156
self.operator = operator
52-
self.field_right = Field(source_name=source_name_right)
57+
self.field_right = Field(source_name=source_name_right) if not is_alias_right else None
5358
self.alias_right = Alias(name=source_name_right) if is_alias_right else None
5459

5560

@@ -60,11 +65,14 @@ def __init__(
6065
operator: Identifier,
6166
value: Union[int, str, StrValue, list, tuple],
6267
is_alias: bool = False,
68+
is_predefined_field: bool = False,
6369
):
64-
self.field = Field(source_name=source_name)
65-
self.alias = None
66-
if is_alias:
67-
self.alias = Alias(name=source_name)
70+
# mapped by platform fields mapping
71+
self.field = Field(source_name=source_name) if not (is_alias or is_predefined_field) else None
72+
# not mapped
73+
self.alias = Alias(name=source_name) if is_alias else None
74+
# mapped by platform predefined fields mapping
75+
self.predefined_field = PredefinedField(name=source_name) if is_predefined_field else None
6876

6977
self.operator = operator
7078
self.values = []
@@ -96,10 +104,13 @@ def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -
96104
self.values.append(value)
97105

98106
def __repr__(self):
99-
if self.field:
100-
return f"{self.field.source_name} {self.operator.token_type} {self.values}"
107+
if self.alias:
108+
return f"{self.alias.name} {self.operator.token_type} {self.values}"
109+
110+
if self.predefined_field:
111+
return f"{self.predefined_field.name} {self.operator.token_type} {self.values}"
101112

102-
return f"{self.alias.name} {self.operator.token_type} {self.values}"
113+
return f"{self.field.source_name} {self.operator.token_type} {self.values}"
103114

104115

105116
class Keyword:

‎uncoder-core/app/translator/core/render.py

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
from app.translator.core.exceptions.parser import UnsupportedOperatorException
3232
from app.translator.core.functions import PlatformFunctions
3333
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
34-
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword
34+
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword, PredefinedField
3535
from app.translator.core.models.functions.base import Function, RenderedFunctions
3636
from app.translator.core.models.identifier import Identifier
3737
from app.translator.core.models.platform_details import PlatformDetails
@@ -218,7 +218,8 @@ class PlatformQueryRender(QueryRender):
218218
field_field_render = BaseFieldFieldRender()
219219
field_value_render = BaseFieldValueRender(or_token=or_token)
220220

221-
raw_log_field_pattern_map: ClassVar[dict[str, str]] = None
221+
predefined_fields_map: ClassVar[dict[str, str]] = {}
222+
raw_log_field_patterns_map: ClassVar[dict[str, str]] = {}
222223

223224
def __init__(self):
224225
super().__init__()
@@ -248,9 +249,23 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
248249

249250
return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]
250251

252+
def map_predefined_field(self, predefined_field: PredefinedField) -> str:
253+
if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)):
254+
if self.is_strict_mapping:
255+
raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name)
256+
257+
return predefined_field.name
258+
259+
return mapped_predefined_field_name
260+
251261
def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
252262
if isinstance(token, FieldValue):
253-
mapped_fields = [token.alias.name] if token.alias else self.map_field(token.field, source_mapping)
263+
if token.alias:
264+
mapped_fields = [token.alias.name]
265+
elif token.predefined_field:
266+
mapped_fields = [self.map_predefined_field(token.predefined_field)]
267+
else:
268+
mapped_fields = self.map_field(token.field, source_mapping)
254269
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
255270
[
256271
self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value)
@@ -365,7 +380,7 @@ def generate_from_raw_query_container(self, query_container: RawQueryContainer)
365380
)
366381

367382
def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
368-
if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type):
383+
if raw_log_field_pattern := self.raw_log_field_patterns_map.get(field_type):
369384
return raw_log_field_pattern.format(field=field)
370385

371386
def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[list]:
@@ -379,7 +394,7 @@ def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping
379394
return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)]
380395

381396
def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str:
382-
if self.raw_log_field_pattern_map is None:
397+
if not self.raw_log_field_patterns_map:
383398
return ""
384399
defined_raw_log_fields = []
385400
for field in fields:

‎uncoder-core/app/translator/core/tokenizer.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -332,12 +332,12 @@ def get_field_tokens_from_func_args( # noqa: PLR0912
332332
if isinstance(arg, Field):
333333
result.append(arg)
334334
elif isinstance(arg, FieldField):
335-
if not arg.alias_left or arg.alias_left.name != arg.field_left.source_name:
335+
if arg.field_left:
336336
result.append(arg.field_left)
337-
if not arg.alias_right or arg.alias_right.name != arg.field_right.source_name:
337+
if arg.field_right:
338338
result.append(arg.field_right)
339339
elif isinstance(arg, FieldValue):
340-
if not arg.alias or arg.alias.name != arg.field.source_name:
340+
if arg.field:
341341
result.append(arg.field)
342342
elif isinstance(arg, GroupByFunction):
343343
result.extend(self.get_field_tokens_from_func_args(args=arg.args))

‎uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ def generate_prefix(self, log_source_signature: LogSourceSignature, functions_pr
219219
return str(log_source_signature)
220220

221221
def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
222-
if isinstance(token, FieldValue):
222+
if isinstance(token, FieldValue) and token.field:
223223
try:
224224
mapped_fields = self.map_field(token.field, source_mapping)
225225
except StrictPlatformException:

‎uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ def finalize_query(
7979
return self.wrap_with_not_supported_functions(rule_str, not_supported_functions)
8080

8181
def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
82-
if isinstance(token, FieldValue):
82+
if isinstance(token, FieldValue) and token.field:
8383
for field in self.map_field(token.field, source_mapping):
8484
self.fields.update({field: f"{{ctx.results.0.hits.hits.0._source.{field}}}"})
8585
return super().apply_token(token, source_mapping)

‎uncoder-core/app/translator/platforms/palo_alto/const.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from app.translator.core.custom_types.predefined_fields import IPLocationType
12
from app.translator.core.models.platform_details import PlatformDetails
23

34
PLATFORM_DETAILS = {"group_id": "cortex", "group_name": "Palo Alto Cortex XSIAM"}
@@ -10,3 +11,15 @@
1011
}
1112

1213
cortex_xql_query_details = PlatformDetails(**CORTEX_XSIAM_XQL_QUERY_DETAILS)
14+
15+
16+
PREDEFINED_FIELDS_MAP = {
17+
IPLocationType.asn: "loc_asn",
18+
IPLocationType.asn_org: "loc_asn_org",
19+
IPLocationType.city: "loc_city",
20+
IPLocationType.continent: "loc_continent",
21+
IPLocationType.country: "loc_country",
22+
IPLocationType.lat_lon: "loc_latlon",
23+
IPLocationType.region: "loc_region",
24+
IPLocationType.timezone: "loc_timezone",
25+
}

‎uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender
3131
from app.translator.core.str_value_manager import StrValue
3232
from app.translator.managers import render_manager
33-
from app.translator.platforms.palo_alto.const import cortex_xql_query_details
33+
from app.translator.platforms.palo_alto.const import PREDEFINED_FIELDS_MAP, cortex_xql_query_details
3434
from app.translator.platforms.palo_alto.functions import CortexXQLFunctions, cortex_xql_functions
3535
from app.translator.platforms.palo_alto.mapping import (
3636
CortexXQLLogSourceSignature,
@@ -167,7 +167,8 @@ class CortexXQLQueryRender(PlatformQueryRender):
167167
details: PlatformDetails = cortex_xql_query_details
168168
mappings: CortexXQLMappings = cortex_xql_mappings
169169
is_strict_mapping = True
170-
raw_log_field_pattern_map: ClassVar[dict[str, str]] = {
170+
predefined_fields_map = PREDEFINED_FIELDS_MAP
171+
raw_log_field_patterns_map: ClassVar[dict[str, str]] = {
171172
"regex": '| alter {field} = regextract(to_json_string(action_evtlog_data_fields)->{field}{{}}, "\\"(.*)\\"")',
172173
"object": '| alter {field_name} = json_extract_scalar({field_object} , "$.{field_path}")',
173174
"list": '| alter {field_name} = arraystring(json_extract_array({field_object} , "$.{field_path}")," ")',
@@ -189,7 +190,7 @@ def init_platform_functions(self) -> None:
189190
self.platform_functions.platform_query_render = self
190191

191192
def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
192-
raw_log_field_pattern = self.raw_log_field_pattern_map.get(field_type)
193+
raw_log_field_pattern = self.raw_log_field_patterns_map.get(field_type)
193194
if raw_log_field_pattern is None:
194195
return
195196
if field_type == "regex":
@@ -206,7 +207,7 @@ def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, fun
206207
return f"{functions_prefix}{log_source_str}"
207208

208209
def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
209-
if isinstance(token, FieldValue):
210+
if isinstance(token, FieldValue) and token.field:
210211
field_name = token.field.source_name
211212
if values_map := SOURCE_MAPPING_TO_FIELD_VALUE_MAP.get(source_mapping.source_id, {}).get(field_name):
212213
values_to_update = []

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy