Skip to content

Commit d10cc06

Browse files
alexvolhaalexvolha
authored
Merge pull request #173 from UncoderIO/gis-8193
mapping flow changes, render unmapped fields to comment
2 parents d9b83b2 + 7d4cc57 commit d10cc06

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+324
-251
lines changed

‎uncoder-core/app/translator/core/exceptions/core.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,14 @@ class BasePlatformException(BaseException):
1010

1111

1212
class StrictPlatformException(BasePlatformException):
13-
field_name: str = None
14-
15-
def __init__(
16-
self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None
17-
):
13+
def __init__(self, platform_name: str, fields: list[str], mapping: Optional[str] = None):
1814
message = (
1915
f"Platform {platform_name} has strict mapping. "
20-
f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping."
16+
f"Source fields: {', '.join(fields)} have no mapping."
2117
f" Mapping file: {mapping}."
2218
if mapping
2319
else ""
2420
)
25-
self.field_name = field_name
2621
super().__init__(message)
2722

2823

‎uncoder-core/app/translator/core/functions.py

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,17 +94,16 @@ def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRe
9494
def render(self, function: Function, source_mapping: SourceMapping) -> str:
9595
raise NotImplementedError
9696

97-
@staticmethod
98-
def map_field(field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
97+
def map_field(self, field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
9998
if isinstance(field, Alias):
10099
return field.name
101100

102-
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
103-
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
104-
if isinstance(mapped_field, list):
105-
mapped_field = mapped_field[0]
101+
if isinstance(field, Field):
102+
mappings = self.manager.platform_functions.platform_query_render.mappings
103+
mapped_fields = mappings.map_field(field, source_mapping)
104+
return mapped_fields[0]
106105

107-
return mapped_field if mapped_field else field.source_name
106+
raise NotSupportedFunctionException
108107

109108

110109
class PlatformFunctionsManager:

‎uncoder-core/app/translator/core/mapping.py

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
from __future__ import annotations
22

33
from abc import ABC, abstractmethod
4-
from typing import Optional, TypeVar
4+
from typing import TYPE_CHECKING, Optional, TypeVar
55

6+
from app.translator.core.exceptions.core import StrictPlatformException
7+
from app.translator.core.models.platform_details import PlatformDetails
68
from app.translator.mappings.utils.load_from_files import LoaderFileMappings
79

10+
if TYPE_CHECKING:
11+
from app.translator.core.models.query_tokens.field import Field
12+
13+
814
DEFAULT_MAPPING_NAME = "default"
915

1016

@@ -85,12 +91,16 @@ def __init__(
8591

8692

8793
class BasePlatformMappings:
94+
details: PlatformDetails = None
95+
96+
is_strict_mapping: bool = False
8897
skip_load_default_mappings: bool = True
8998
extend_default_mapping_with_all_fields: bool = False
9099

91-
def __init__(self, platform_dir: str):
100+
def __init__(self, platform_dir: str, platform_details: PlatformDetails):
92101
self._loader = LoaderFileMappings()
93102
self._platform_dir = platform_dir
103+
self.details = platform_details
94104
self._source_mappings = self.prepare_mapping()
95105

96106
def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None:
@@ -148,6 +158,32 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
148158
def default_mapping(self) -> SourceMapping:
149159
return self._source_mappings[DEFAULT_MAPPING_NAME]
150160

161+
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
162+
unmapped = []
163+
for field in field_tokens:
164+
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
165+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
166+
if not mapped_field and field.source_name not in unmapped:
167+
unmapped.append(field.source_name)
168+
169+
if self.is_strict_mapping and unmapped:
170+
raise StrictPlatformException(
171+
platform_name=self.details.name, fields=unmapped, mapping=source_mapping.source_id
172+
)
173+
174+
return unmapped
175+
176+
@staticmethod
177+
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
178+
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
179+
# field can be mapped to corresponding platform field name or list of platform field names
180+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
181+
182+
if isinstance(mapped_field, str):
183+
mapped_field = [mapped_field]
184+
185+
return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]
186+
151187

152188
class BaseCommonPlatformMappings(ABC, BasePlatformMappings):
153189
def prepare_mapping(self) -> dict[str, SourceMapping]:

‎uncoder-core/app/translator/core/render.py

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ class QueryRender(ABC):
184184
details: PlatformDetails = None
185185
is_single_line_comment: bool = False
186186
unsupported_functions_text = "Unsupported functions were excluded from the result query:"
187+
unmapped_fields_text = "Unmapped fields: "
187188

188189
platform_functions: PlatformFunctions = None
189190

@@ -206,6 +207,11 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions:
206207

207208
return query
208209

210+
def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[str]]) -> str:
211+
if fields:
212+
return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{', '.join(fields)}")
213+
return query
214+
209215
def wrap_with_comment(self, value: str) -> str:
210216
return f"{self.comment_symbol} {value}"
211217

@@ -216,7 +222,6 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta
216222

217223
class PlatformQueryRender(QueryRender):
218224
mappings: BasePlatformMappings = None
219-
is_strict_mapping: bool = False
220225

221226
or_token = "or"
222227
and_token = "and"
@@ -247,22 +252,10 @@ def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], fu
247252
def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions:
248253
return self.platform_functions.render(functions, source_mapping)
249254

250-
def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
251-
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
252-
# field can be mapped to corresponding platform field name or list of platform field names
253-
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
254-
if not mapped_field and self.is_strict_mapping:
255-
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
256-
257-
if isinstance(mapped_field, str):
258-
mapped_field = [mapped_field]
259-
260-
return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]
261-
262255
def map_predefined_field(self, predefined_field: PredefinedField) -> str:
263256
if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)):
264-
if self.is_strict_mapping:
265-
raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name)
257+
if self.mappings.is_strict_mapping:
258+
raise StrictPlatformException(platform_name=self.details.name, fields=[predefined_field.name])
266259

267260
return predefined_field.name
268261

@@ -275,7 +268,7 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
275268
elif token.predefined_field:
276269
mapped_fields = [self.map_predefined_field(token.predefined_field)]
277270
else:
278-
mapped_fields = self.map_field(token.field, source_mapping)
271+
mapped_fields = self.mappings.map_field(token.field, source_mapping)
279272
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
280273
[
281274
self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value)
@@ -285,9 +278,13 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
285278
return self.group_token % joined if len(mapped_fields) > 1 else joined
286279
if isinstance(token, FieldField):
287280
alias_left, field_left = token.alias_left, token.field_left
288-
mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping)
281+
mapped_fields_left = (
282+
[alias_left.name] if alias_left else self.mappings.map_field(field_left, source_mapping)
283+
)
289284
alias_right, field_right = token.alias_right, token.field_right
290-
mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping)
285+
mapped_fields_right = (
286+
[alias_right.name] if alias_right else self.mappings.map_field(field_right, source_mapping)
287+
)
291288
cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right))
292289
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
293290
[
@@ -311,14 +308,9 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
311308

312309
def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str:
313310
result_values = []
314-
unmapped_fields = set()
315311
for token in tokens:
316-
try:
317-
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
318-
except StrictPlatformException as err:
319-
unmapped_fields.add(err.field_name)
320-
if unmapped_fields:
321-
raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields))
312+
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
313+
322314
return "".join(result_values)
323315

324316
def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str:
@@ -351,11 +343,13 @@ def finalize_query(
351343
meta_info: Optional[MetaInfoContainer] = None,
352344
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
353345
not_supported_functions: Optional[list] = None,
346+
unmapped_fields: Optional[list[str]] = None,
354347
*args, # noqa: ARG002
355348
**kwargs, # noqa: ARG002
356349
) -> str:
357350
query = self._join_query_parts(prefix, query, functions)
358351
query = self.wrap_with_meta_info(query, meta_info)
352+
query = self.wrap_with_unmapped_fields(query, unmapped_fields)
359353
return self.wrap_with_not_supported_functions(query, not_supported_functions)
360354

361355
@staticmethod
@@ -417,8 +411,10 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
417411
mapped_field = source_mapping.fields_mapping.get_platform_field_name(
418412
generic_field_name=generic_field_name
419413
)
420-
if not mapped_field and self.is_strict_mapping:
421-
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
414+
if not mapped_field and self.mappings.is_strict_mapping:
415+
raise StrictPlatformException(
416+
platform_name=self.details.name, fields=[field.source_name], mapping=source_mapping.source_id
417+
)
422418
if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
423419
for prefix in prefix_list:
424420
if prefix not in defined_raw_log_fields:
@@ -428,6 +424,9 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
428424
def _generate_from_tokenized_query_container_by_source_mapping(
429425
self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping
430426
) -> str:
427+
unmapped_fields = self.mappings.check_fields_mapping_existence(
428+
query_container.meta_info.query_fields, source_mapping
429+
)
431430
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
432431
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)
433432

@@ -443,6 +442,7 @@ def _generate_from_tokenized_query_container_by_source_mapping(
443442
query=query,
444443
functions=rendered_functions.rendered,
445444
not_supported_functions=not_supported_functions,
445+
unmapped_fields=unmapped_fields,
446446
meta_info=query_container.meta_info,
447447
source_mapping=source_mapping,
448448
)

‎uncoder-core/app/translator/platforms/athena/const.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@
99
"alt_platform_name": "OCSF",
1010
}
1111

12-
athena_details = PlatformDetails(**ATHENA_QUERY_DETAILS)
12+
athena_query_details = PlatformDetails(**ATHENA_QUERY_DETAILS)

‎uncoder-core/app/translator/platforms/athena/mapping.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import Optional
22

33
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
4+
from app.translator.platforms.athena.const import athena_query_details
45

56

67
class AthenaLogSourceSignature(LogSourceSignature):
@@ -40,4 +41,4 @@ def get_suitable_source_mappings(self, field_names: list[str], table: Optional[s
4041
return suitable_source_mappings
4142

4243

43-
athena_mappings = AthenaMappings(platform_dir="athena")
44+
athena_query_mappings = AthenaMappings(platform_dir="athena", platform_details=athena_query_details)

‎uncoder-core/app/translator/platforms/athena/parsers/athena.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@
1818

1919
from app.translator.core.models.platform_details import PlatformDetails
2020
from app.translator.managers import parser_manager
21-
from app.translator.platforms.athena.const import athena_details
22-
from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings
21+
from app.translator.platforms.athena.const import athena_query_details
22+
from app.translator.platforms.athena.mapping import AthenaMappings, athena_query_mappings
2323
from app.translator.platforms.base.sql.parsers.sql import SqlQueryParser
2424

2525

2626
@parser_manager.register_supported_by_roota
2727
class AthenaQueryParser(SqlQueryParser):
28-
details: PlatformDetails = athena_details
29-
mappings: AthenaMappings = athena_mappings
28+
details: PlatformDetails = athena_query_details
29+
mappings: AthenaMappings = athena_query_mappings
3030
query_delimiter_pattern = r"\sFROM\s\S*\sWHERE\s"
3131
table_pattern = r"\sFROM\s(?P<table>[a-zA-Z\.\-\*]+)\sWHERE\s"

‎uncoder-core/app/translator/platforms/athena/renders/athena.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,19 @@
1919

2020
from app.translator.core.models.platform_details import PlatformDetails
2121
from app.translator.managers import render_manager
22-
from app.translator.platforms.athena.const import athena_details
23-
from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings
22+
from app.translator.platforms.athena.const import athena_query_details
23+
from app.translator.platforms.athena.mapping import AthenaMappings, athena_query_mappings
2424
from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender
2525

2626

2727
class AthenaFieldValueRender(SqlFieldValueRender):
28-
details: PlatformDetails = athena_details
28+
details: PlatformDetails = athena_query_details
2929

3030

3131
@render_manager.register
3232
class AthenaQueryRender(SqlQueryRender):
33-
details: PlatformDetails = athena_details
34-
mappings: AthenaMappings = athena_mappings
33+
details: PlatformDetails = athena_query_details
34+
mappings: AthenaMappings = athena_query_mappings
3535

3636
or_token = "OR"
3737

‎uncoder-core/app/translator/platforms/athena/renders/athena_cti.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@
2020
from app.translator.core.models.platform_details import PlatformDetails
2121
from app.translator.core.render_cti import RenderCTI
2222
from app.translator.managers import render_cti_manager
23-
from app.translator.platforms.athena.const import athena_details
23+
from app.translator.platforms.athena.const import athena_query_details
2424
from app.translator.platforms.athena.mappings.athena_cti import DEFAULT_ATHENA_MAPPING
2525

2626

2727
@render_cti_manager.register
2828
class AthenaCTI(RenderCTI):
29-
details: PlatformDetails = athena_details
29+
details: PlatformDetails = athena_query_details
3030

3131
field_value_template: str = "{key} = '{value}'"
3232
or_operator: str = " OR "

‎uncoder-core/app/translator/platforms/base/aql/mapping.py

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,3 @@ def get_suitable_source_mappings(
9090
suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]]
9191

9292
return suitable_source_mappings
93-
94-
95-
aql_mappings = AQLMappings(platform_dir="qradar")

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy