Skip to content

Commit 14ec9a0

Browse files
alexvolhaalexvolha
committed
add unmapped fields to comment
1 parent a4e8cda commit 14ec9a0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+317
-194
lines changed

‎uncoder-core/app/translator/core/mapping.py

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
from __future__ import annotations
22

33
from abc import ABC, abstractmethod
4-
from typing import Optional, TypeVar
4+
from typing import TYPE_CHECKING, Optional, TypeVar
55

6+
from app.translator.core.exceptions.core import StrictPlatformException
7+
from app.translator.core.models.platform_details import PlatformDetails
68
from app.translator.mappings.utils.load_from_files import LoaderFileMappings
79

10+
if TYPE_CHECKING:
11+
from app.translator.core.models.query_tokens.field import Field
12+
13+
814
DEFAULT_MAPPING_NAME = "default"
915

1016

@@ -85,12 +91,16 @@ def __init__(
8591

8692

8793
class BasePlatformMappings:
94+
details: PlatformDetails = None
95+
96+
is_strict_mapping: bool = False
8897
skip_load_default_mappings: bool = True
8998
extend_default_mapping_with_all_fields: bool = False
9099

91-
def __init__(self, platform_dir: str):
100+
def __init__(self, platform_dir: str, platform_details: PlatformDetails):
92101
self._loader = LoaderFileMappings()
93102
self._platform_dir = platform_dir
103+
self.details = platform_details
94104
self._source_mappings = self.prepare_mapping()
95105

96106
def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None:
@@ -148,6 +158,29 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
148158
def default_mapping(self) -> SourceMapping:
149159
return self._source_mappings[DEFAULT_MAPPING_NAME]
150160

161+
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[Field]:
162+
not_mapped = []
163+
for field in field_tokens:
164+
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
165+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
166+
if not mapped_field:
167+
if self.is_strict_mapping:
168+
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
169+
not_mapped.append(field)
170+
171+
return not_mapped
172+
173+
@staticmethod
174+
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
175+
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
176+
# field can be mapped to corresponding platform field name or list of platform field names
177+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
178+
179+
if isinstance(mapped_field, str):
180+
mapped_field = [mapped_field]
181+
182+
return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]
183+
151184

152185
class BaseCommonPlatformMappings(ABC, BasePlatformMappings):
153186
def prepare_mapping(self) -> dict[str, SourceMapping]:

‎uncoder-core/app/translator/core/render.py

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ class QueryRender(ABC):
184184
details: PlatformDetails = None
185185
is_single_line_comment: bool = False
186186
unsupported_functions_text = "Unsupported functions were excluded from the result query:"
187+
unmapped_fields_text = "Unmapped fields: "
187188

188189
platform_functions: PlatformFunctions = None
189190

@@ -206,6 +207,12 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions:
206207

207208
return query
208209

210+
def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[Field]]) -> str:
211+
if fields:
212+
joined = ", ".join(field.source_name for field in fields)
213+
return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{joined}")
214+
return query
215+
209216
def wrap_with_comment(self, value: str) -> str:
210217
return f"{self.comment_symbol} {value}"
211218

@@ -216,7 +223,6 @@ def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryConta
216223

217224
class PlatformQueryRender(QueryRender):
218225
mappings: BasePlatformMappings = None
219-
is_strict_mapping: bool = False
220226

221227
or_token = "or"
222228
and_token = "and"
@@ -247,21 +253,9 @@ def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], fu
247253
def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions:
248254
return self.platform_functions.render(functions, source_mapping)
249255

250-
def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
251-
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
252-
# field can be mapped to corresponding platform field name or list of platform field names
253-
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
254-
if not mapped_field and self.is_strict_mapping:
255-
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
256-
257-
if isinstance(mapped_field, str):
258-
mapped_field = [mapped_field]
259-
260-
return mapped_field if mapped_field else [generic_field_name] if generic_field_name else [field.source_name]
261-
262256
def map_predefined_field(self, predefined_field: PredefinedField) -> str:
263257
if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)):
264-
if self.is_strict_mapping:
258+
if self.mappings.is_strict_mapping:
265259
raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name)
266260

267261
return predefined_field.name
@@ -275,7 +269,7 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
275269
elif token.predefined_field:
276270
mapped_fields = [self.map_predefined_field(token.predefined_field)]
277271
else:
278-
mapped_fields = self.map_field(token.field, source_mapping)
272+
mapped_fields = self.mappings.map_field(token.field, source_mapping)
279273
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
280274
[
281275
self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value)
@@ -285,9 +279,13 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
285279
return self.group_token % joined if len(mapped_fields) > 1 else joined
286280
if isinstance(token, FieldField):
287281
alias_left, field_left = token.alias_left, token.field_left
288-
mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping)
282+
mapped_fields_left = (
283+
[alias_left.name] if alias_left else self.mappings.map_field(field_left, source_mapping)
284+
)
289285
alias_right, field_right = token.alias_right, token.field_right
290-
mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping)
286+
mapped_fields_right = (
287+
[alias_right.name] if alias_right else self.mappings.map_field(field_right, source_mapping)
288+
)
291289
cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right))
292290
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
293291
[
@@ -351,11 +349,13 @@ def finalize_query(
351349
meta_info: Optional[MetaInfoContainer] = None,
352350
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
353351
not_supported_functions: Optional[list] = None,
352+
unmapped_fields: Optional[list[Field]] = None,
354353
*args, # noqa: ARG002
355354
**kwargs, # noqa: ARG002
356355
) -> str:
357356
query = self._join_query_parts(prefix, query, functions)
358357
query = self.wrap_with_meta_info(query, meta_info)
358+
query = self.wrap_with_unmapped_fields(query, unmapped_fields)
359359
return self.wrap_with_not_supported_functions(query, not_supported_functions)
360360

361361
@staticmethod
@@ -417,7 +417,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
417417
mapped_field = source_mapping.fields_mapping.get_platform_field_name(
418418
generic_field_name=generic_field_name
419419
)
420-
if not mapped_field and self.is_strict_mapping:
420+
if not mapped_field and self.mappings.is_strict_mapping:
421421
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
422422
if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
423423
for prefix in prefix_list:
@@ -428,6 +428,9 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
428428
def _generate_from_tokenized_query_container_by_source_mapping(
429429
self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping
430430
) -> str:
431+
unmapped_fields = self.mappings.check_fields_mapping_existence(
432+
query_container.meta_info.query_fields, source_mapping
433+
)
431434
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
432435
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)
433436

@@ -443,6 +446,7 @@ def _generate_from_tokenized_query_container_by_source_mapping(
443446
query=query,
444447
functions=rendered_functions.rendered,
445448
not_supported_functions=not_supported_functions,
449+
unmapped_fields=unmapped_fields,
446450
meta_info=query_container.meta_info,
447451
source_mapping=source_mapping,
448452
)

‎uncoder-core/app/translator/platforms/athena/const.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,4 @@
99
"alt_platform_name": "OCSF",
1010
}
1111

12-
athena_details = PlatformDetails(**ATHENA_QUERY_DETAILS)
12+
athena_query_details = PlatformDetails(**ATHENA_QUERY_DETAILS)

‎uncoder-core/app/translator/platforms/athena/mapping.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import Optional
22

33
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
4+
from app.translator.platforms.athena.const import athena_query_details
45

56

67
class AthenaLogSourceSignature(LogSourceSignature):
@@ -40,4 +41,4 @@ def get_suitable_source_mappings(self, field_names: list[str], table: Optional[s
4041
return suitable_source_mappings
4142

4243

43-
athena_mappings = AthenaMappings(platform_dir="athena")
44+
athena_query_mappings = AthenaMappings(platform_dir="athena", platform_details=athena_query_details)

‎uncoder-core/app/translator/platforms/athena/parsers/athena.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@
1818

1919
from app.translator.core.models.platform_details import PlatformDetails
2020
from app.translator.managers import parser_manager
21-
from app.translator.platforms.athena.const import athena_details
22-
from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings
21+
from app.translator.platforms.athena.const import athena_query_details
22+
from app.translator.platforms.athena.mapping import AthenaMappings, athena_query_mappings
2323
from app.translator.platforms.base.sql.parsers.sql import SqlQueryParser
2424

2525

2626
@parser_manager.register_supported_by_roota
2727
class AthenaQueryParser(SqlQueryParser):
28-
details: PlatformDetails = athena_details
29-
mappings: AthenaMappings = athena_mappings
28+
details: PlatformDetails = athena_query_details
29+
mappings: AthenaMappings = athena_query_mappings
3030
query_delimiter_pattern = r"\sFROM\s\S*\sWHERE\s"
3131
table_pattern = r"\sFROM\s(?P<table>[a-zA-Z\.\-\*]+)\sWHERE\s"

‎uncoder-core/app/translator/platforms/athena/renders/athena.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,19 @@
1919

2020
from app.translator.core.models.platform_details import PlatformDetails
2121
from app.translator.managers import render_manager
22-
from app.translator.platforms.athena.const import athena_details
23-
from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings
22+
from app.translator.platforms.athena.const import athena_query_details
23+
from app.translator.platforms.athena.mapping import AthenaMappings, athena_query_mappings
2424
from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender
2525

2626

2727
class AthenaFieldValueRender(SqlFieldValueRender):
28-
details: PlatformDetails = athena_details
28+
details: PlatformDetails = athena_query_details
2929

3030

3131
@render_manager.register
3232
class AthenaQueryRender(SqlQueryRender):
33-
details: PlatformDetails = athena_details
34-
mappings: AthenaMappings = athena_mappings
33+
details: PlatformDetails = athena_query_details
34+
mappings: AthenaMappings = athena_query_mappings
3535

3636
or_token = "OR"
3737

‎uncoder-core/app/translator/platforms/athena/renders/athena_cti.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@
2020
from app.translator.core.models.platform_details import PlatformDetails
2121
from app.translator.core.render_cti import RenderCTI
2222
from app.translator.managers import render_cti_manager
23-
from app.translator.platforms.athena.const import athena_details
23+
from app.translator.platforms.athena.const import athena_query_details
2424
from app.translator.platforms.athena.mappings.athena_cti import DEFAULT_ATHENA_MAPPING
2525

2626

2727
@render_cti_manager.register
2828
class AthenaCTI(RenderCTI):
29-
details: PlatformDetails = athena_details
29+
details: PlatformDetails = athena_query_details
3030

3131
field_value_template: str = "{key} = '{value}'"
3232
or_operator: str = " OR "

‎uncoder-core/app/translator/platforms/base/aql/mapping.py

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,3 @@ def get_suitable_source_mappings(
9090
suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]]
9191

9292
return suitable_source_mappings
93-
94-
95-
aql_mappings = AQLMappings(platform_dir="qradar")

‎uncoder-core/app/translator/platforms/base/aql/parsers/aql.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,12 @@
2626
from app.translator.platforms.base.aql.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, TABLE_GROUP_PATTERN
2727
from app.translator.platforms.base.aql.functions import AQLFunctions, aql_functions
2828
from app.translator.platforms.base.aql.log_source_map import LOG_SOURCE_FUNCTIONS_MAP
29-
from app.translator.platforms.base.aql.mapping import AQLMappings, aql_mappings
3029
from app.translator.platforms.base.aql.tokenizer import AQLTokenizer
3130
from app.translator.tools.utils import get_match_group
3231

3332

3433
class AQLQueryParser(PlatformQueryParser):
3534
tokenizer: AQLTokenizer = AQLTokenizer(aql_functions)
36-
mappings: AQLMappings = aql_mappings
3735
platform_functions: AQLFunctions = aql_functions
3836

3937
log_source_functions = ("LOGSOURCENAME", "LOGSOURCEGROUPNAME")

‎uncoder-core/app/translator/platforms/base/aql/renders/aql.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from app.translator.core.custom_types.values import ValueType
2424
from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
2525
from app.translator.core.str_value_manager import StrValue
26-
from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings
26+
from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature
2727
from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager
2828

2929

@@ -121,8 +121,6 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
121121

122122

123123
class AQLQueryRender(PlatformQueryRender):
124-
mappings: AQLMappings = aql_mappings
125-
126124
or_token = "OR"
127125
and_token = "AND"
128126
not_token = "NOT"

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy