Skip to content

Commit 83c12c8

Browse files
alexvolhaalexvolha
committed
render unmapped fields comment
1 parent 14ec9a0 commit 83c12c8

File tree

15 files changed

+30
-83
lines changed

15 files changed

+30
-83
lines changed

‎uncoder-core/app/translator/core/exceptions/core.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,14 @@ class BasePlatformException(BaseException):
1010

1111

1212
class StrictPlatformException(BasePlatformException):
13-
field_name: str = None
14-
15-
def __init__(
16-
self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None
17-
):
13+
def __init__(self, platform_name: str, fields: list[str], mapping: Optional[str] = None):
1814
message = (
1915
f"Platform {platform_name} has strict mapping. "
20-
f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping."
16+
f"Source fields: {', '.join(fields)} have no mapping."
2117
f" Mapping file: {mapping}."
2218
if mapping
2319
else ""
2420
)
25-
self.field_name = field_name
2621
super().__init__(message)
2722

2823

‎uncoder-core/app/translator/core/mapping.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -158,17 +158,18 @@ def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
158158
def default_mapping(self) -> SourceMapping:
159159
return self._source_mappings[DEFAULT_MAPPING_NAME]
160160

161-
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[Field]:
162-
not_mapped = []
161+
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
162+
unmapped = []
163163
for field in field_tokens:
164164
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
165165
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
166-
if not mapped_field:
167-
if self.is_strict_mapping:
168-
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
169-
not_mapped.append(field)
166+
if not mapped_field and field.source_name not in unmapped:
167+
unmapped.append(field.source_name)
170168

171-
return not_mapped
169+
if self.is_strict_mapping and unmapped:
170+
raise StrictPlatformException(platform_name=self.details.name, fields=unmapped)
171+
172+
return unmapped
172173

173174
@staticmethod
174175
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:

‎uncoder-core/app/translator/core/render.py

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -207,10 +207,9 @@ def wrap_with_not_supported_functions(self, query: str, not_supported_functions:
207207

208208
return query
209209

210-
def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[Field]]) -> str:
210+
def wrap_with_unmapped_fields(self, query: str, fields: Optional[list[str]]) -> str:
211211
if fields:
212-
joined = ", ".join(field.source_name for field in fields)
213-
return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{joined}")
212+
return query + "\n\n" + self.wrap_with_comment(f"{self.unmapped_fields_text}{', '.join(fields)}")
214213
return query
215214

216215
def wrap_with_comment(self, value: str) -> str:
@@ -256,7 +255,7 @@ def generate_functions(self, functions: list[Function], source_mapping: SourceMa
256255
def map_predefined_field(self, predefined_field: PredefinedField) -> str:
257256
if not (mapped_predefined_field_name := self.predefined_fields_map.get(predefined_field.name)):
258257
if self.mappings.is_strict_mapping:
259-
raise StrictPlatformException(field_name=predefined_field.name, platform_name=self.details.name)
258+
raise StrictPlatformException(platform_name=self.details.name, fields=[predefined_field.name])
260259

261260
return predefined_field.name
262261

@@ -309,14 +308,9 @@ def apply_token(self, token: QUERY_TOKEN_TYPE, source_mapping: SourceMapping) ->
309308

310309
def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str:
311310
result_values = []
312-
unmapped_fields = set()
313311
for token in tokens:
314-
try:
315-
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
316-
except StrictPlatformException as err:
317-
unmapped_fields.add(err.field_name)
318-
if unmapped_fields:
319-
raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields))
312+
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
313+
320314
return "".join(result_values)
321315

322316
def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str:
@@ -349,7 +343,7 @@ def finalize_query(
349343
meta_info: Optional[MetaInfoContainer] = None,
350344
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
351345
not_supported_functions: Optional[list] = None,
352-
unmapped_fields: Optional[list[Field]] = None,
346+
unmapped_fields: Optional[list[str]] = None,
353347
*args, # noqa: ARG002
354348
**kwargs, # noqa: ARG002
355349
) -> str:
@@ -418,7 +412,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
418412
generic_field_name=generic_field_name
419413
)
420414
if not mapped_field and self.mappings.is_strict_mapping:
421-
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
415+
raise StrictPlatformException(platform_name=self.details.name, fields=[field.source_name])
422416
if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
423417
for prefix in prefix_list:
424418
if prefix not in defined_raw_log_fields:

‎uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
from app.translator.core.mapping import SourceMapping
2525
from app.translator.core.models.platform_details import PlatformDetails
2626
from app.translator.core.models.query_container import MetaInfoContainer
27-
from app.translator.core.models.query_tokens.field import Field
2827
from app.translator.managers import render_manager
2928
from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details
3029
from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_rule_mappings
@@ -112,7 +111,7 @@ def finalize_query(
112111
meta_info: Optional[MetaInfoContainer] = None,
113112
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
114113
not_supported_functions: Optional[list] = None, # ,
115-
unmapped_fields: Optional[list[Field]] = None,
114+
unmapped_fields: Optional[list[str]] = None,
116115
*args, # noqa: ARG002
117116
**kwargs, # noqa: ARG002
118117
) -> str:

‎uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
from app.translator.core.mitre import MitreConfig
2626
from app.translator.core.models.platform_details import PlatformDetails
2727
from app.translator.core.models.query_container import MetaInfoContainer
28-
from app.translator.core.models.query_tokens.field import Field
2928
from app.translator.managers import render_manager
3029
from app.translator.platforms.base.lucene.mapping import LuceneMappings
3130
from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_DETECTION_RULE, elasticsearch_rule_details
@@ -88,7 +87,7 @@ def finalize_query(
8887
meta_info: Optional[MetaInfoContainer] = None,
8988
source_mapping: Optional[SourceMapping] = None,
9089
not_supported_functions: Optional[list] = None,
91-
unmapped_fields: Optional[list[Field]] = None,
90+
unmapped_fields: Optional[list[str]] = None,
9291
*args, # noqa: ARG002
9392
**kwargs, # noqa: ARG002
9493
) -> str:

‎uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
from app.translator.core.mapping import SourceMapping
2424
from app.translator.core.models.platform_details import PlatformDetails
2525
from app.translator.core.models.query_container import MetaInfoContainer
26-
from app.translator.core.models.query_tokens.field import Field
2726
from app.translator.managers import render_manager
2827
from app.translator.platforms.base.lucene.mapping import LuceneMappings
2928
from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_ALERT, elastalert_details
@@ -61,7 +60,7 @@ def finalize_query(
6160
meta_info: Optional[MetaInfoContainer] = None,
6261
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
6362
not_supported_functions: Optional[list] = None,
64-
unmapped_fields: Optional[list[Field]] = None,
63+
unmapped_fields: Optional[list[str]] = None,
6564
*args, # noqa: ARG002
6665
**kwargs, # noqa: ARG002
6766
) -> str:

‎uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
from app.translator.core.mapping import SourceMapping
2525
from app.translator.core.models.platform_details import PlatformDetails
2626
from app.translator.core.models.query_container import MetaInfoContainer
27-
from app.translator.core.models.query_tokens.field import Field
2827
from app.translator.managers import render_manager
2928
from app.translator.platforms.base.lucene.mapping import LuceneMappings
3029
from app.translator.platforms.elasticsearch.const import KIBANA_RULE, KIBANA_SEARCH_SOURCE_JSON, kibana_rule_details
@@ -57,7 +56,7 @@ def finalize_query(
5756
meta_info: Optional[MetaInfoContainer] = None,
5857
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
5958
not_supported_functions: Optional[list] = None,
60-
unmapped_fields: Optional[list[Field]] = None,
59+
unmapped_fields: Optional[list[str]] = None,
6160
*args, # noqa: ARG002
6261
**kwargs, # noqa: ARG002
6362
) -> str:

‎uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
from app.translator.core.mapping import SourceMapping
2525
from app.translator.core.models.platform_details import PlatformDetails
2626
from app.translator.core.models.query_container import MetaInfoContainer
27-
from app.translator.core.models.query_tokens.field import Field
2827
from app.translator.managers import render_manager
2928
from app.translator.platforms.base.lucene.mapping import LuceneMappings
3029
from app.translator.platforms.elasticsearch.const import XPACK_WATCHER_RULE, xpack_watcher_details
@@ -57,7 +56,7 @@ def finalize_query(
5756
meta_info: Optional[MetaInfoContainer] = None,
5857
source_mapping: Optional[SourceMapping] = None,
5958
not_supported_functions: Optional[list] = None,
60-
unmapped_fields: Optional[list[Field]] = None,
59+
unmapped_fields: Optional[list[str]] = None,
6160
*args, # noqa: ARG002
6261
**kwargs, # noqa: ARG002
6362
) -> str:

‎uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
from app.translator.core.mapping import SourceMapping
2727
from app.translator.core.models.platform_details import PlatformDetails
2828
from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer
29-
from app.translator.core.models.query_tokens.field import Field
3029
from app.translator.core.models.query_tokens.field_value import FieldValue
3130
from app.translator.core.models.query_tokens.identifier import Identifier
3231
from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
@@ -304,7 +303,7 @@ def finalize_query(
304303
meta_info: Optional[MetaInfoContainer] = None,
305304
source_mapping: Optional[SourceMapping] = None, # noqa: ARG002
306305
not_supported_functions: Optional[list] = None,
307-
unmapped_fields: Optional[list[Field]] = None,
306+
unmapped_fields: Optional[list[str]] = None,
308307
fields: Optional[set[str]] = None,
309308
*args, # noqa: ARG002
310309
**kwargs, # noqa: ARG002

‎uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
from app.translator.core.mapping import SourceMapping
2626
from app.translator.core.models.platform_details import PlatformDetails
2727
from app.translator.core.models.query_container import MetaInfoContainer
28-
from app.translator.core.models.query_tokens.field import Field
2928
from app.translator.managers import render_manager
3029
from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details
3130
from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager
@@ -66,7 +65,7 @@ def finalize_query(
6665
meta_info: Optional[MetaInfoContainer] = None,
6766
source_mapping: Optional[SourceMapping] = None,
6867
not_supported_functions: Optional[list] = None,
69-
unmapped_fields: Optional[list[Field]] = None,
68+
unmapped_fields: Optional[list[str]] = None,
7069
*args, # noqa: ARG002
7170
**kwargs, # noqa: ARG002
7271
) -> str:

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy