Skip to content

Commit b94266a

Browse files
alexvolhaalexvolha
committed
FieldField class, renders improvements
1 parent 7606f21 commit b94266a

File tree

37 files changed

+221
-187
lines changed

37 files changed

+221
-187
lines changed

‎uncoder-core/app/translator/core/const.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
from typing import Union
2+
3+
from app.translator.core.models.field import Alias, Field, FieldValue, Keyword
4+
from app.translator.core.models.identifier import Identifier
5+
6+
TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias]

‎uncoder-core/app/translator/core/context_vars.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
11
from contextvars import ContextVar
2+
from typing import Optional
23

34
return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False)
45
"""Set to True to return only first query if rendered multiple options"""
6+
7+
wrap_query_with_meta_info_ctx_var: ContextVar[bool] = ContextVar("wrap_query_with_meta_info_ctx_var", default=True)
8+
"""Set to False not to wrap query with meta info commentary"""
9+
10+
preset_log_source_str_ctx_var: ContextVar[Optional[str]] = ContextVar("preset_log_source_str_ctx_var", default=None)

‎uncoder-core/app/translator/core/models/field.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,22 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma
3737
self.__generic_names_map = generic_names_map
3838

3939

40+
class FieldField:
41+
def __init__(
42+
self,
43+
source_name_left: str,
44+
operator: Identifier,
45+
source_name_right: str,
46+
is_alias_left: bool = False,
47+
is_alias_right: bool = False,
48+
):
49+
self.field_left = Field(source_name=source_name_left)
50+
self.alias_left = Alias(name=source_name_left) if is_alias_left else None
51+
self.operator = operator
52+
self.field_right = Field(source_name=source_name_right)
53+
self.alias_right = Alias(name=source_name_right) if is_alias_right else None
54+
55+
4056
class FieldValue:
4157
def __init__(
4258
self,

‎uncoder-core/app/translator/core/models/query_container.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
from datetime import datetime
44
from typing import Optional
55

6+
from app.translator.core.const import TOKEN_TYPE
67
from app.translator.core.custom_types.meta_info import SeverityType
78
from app.translator.core.mapping import DEFAULT_MAPPING_NAME
89
from app.translator.core.models.field import Field
910
from app.translator.core.models.functions.base import ParsedFunctions
10-
from app.translator.core.tokenizer import TOKEN_TYPE
1111

1212

1313
class MetaInfoContainer:

‎uncoder-core/app/translator/core/parser.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from abc import ABC, abstractmethod
2121
from typing import Union
2222

23+
from app.translator.core.const import TOKEN_TYPE
2324
from app.translator.core.exceptions.parser import TokenizerGeneralException
2425
from app.translator.core.functions import PlatformFunctions
2526
from app.translator.core.mapping import BasePlatformMappings, SourceMapping
@@ -28,7 +29,7 @@
2829
from app.translator.core.models.identifier import Identifier
2930
from app.translator.core.models.platform_details import PlatformDetails
3031
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
31-
from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer
32+
from app.translator.core.tokenizer import QueryTokenizer
3233

3334

3435
class QueryParser(ABC):

‎uncoder-core/app/translator/core/render.py

Lines changed: 57 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -16,36 +16,36 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19-
19+
import itertools
2020
from abc import ABC, abstractmethod
2121
from collections.abc import Callable
2222
from typing import ClassVar, Optional, Union
2323

2424
from app.translator.const import DEFAULT_VALUE_TYPE
25-
from app.translator.core.context_vars import return_only_first_query_ctx_var
25+
from app.translator.core.const import TOKEN_TYPE
26+
from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var
2627
from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType
2728
from app.translator.core.custom_types.values import ValueType
2829
from app.translator.core.escape_manager import EscapeManager
2930
from app.translator.core.exceptions.core import NotImplementedException, StrictPlatformException
3031
from app.translator.core.exceptions.parser import UnsupportedOperatorException
3132
from app.translator.core.functions import PlatformFunctions
3233
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
33-
from app.translator.core.models.field import Field, FieldValue, Keyword
34+
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword
3435
from app.translator.core.models.functions.base import Function, RenderedFunctions
3536
from app.translator.core.models.identifier import Identifier
3637
from app.translator.core.models.platform_details import PlatformDetails
3738
from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer
3839
from app.translator.core.str_value_manager import StrValue, StrValueManager
39-
from app.translator.core.tokenizer import TOKEN_TYPE
4040

4141

42-
class BaseQueryFieldValue(ABC):
42+
class BaseFieldValueRender(ABC):
4343
details: PlatformDetails = None
4444
escape_manager: EscapeManager = None
4545
str_value_manager: StrValueManager = None
4646

4747
def __init__(self, or_token: str):
48-
self.field_value: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = {
48+
self.modifiers_map: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = {
4949
OperatorType.EQ: self.equal_modifier,
5050
OperatorType.NOT_EQ: self.not_equal_modifier,
5151
OperatorType.LT: self.less_modifier,
@@ -155,11 +155,20 @@ def apply_value(self, value: Union[str, int], value_type: str = ValueType.value)
155155
return self.escape_manager.escape(value, value_type)
156156

157157
def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VALUE_TYPE) -> str:
158-
if modifier_function := self.field_value.get(operator.token_type):
158+
if modifier_function := self.modifiers_map.get(operator.token_type):
159159
return modifier_function(field, value)
160160
raise UnsupportedOperatorException(operator.token_type)
161161

162162

163+
class BaseFieldFieldRender(ABC):
164+
operators_map: ClassVar[dict[str, str]] = {}
165+
166+
def apply_field_field(self, field_left: str, operator: Identifier, field_right: str) -> str:
167+
if mapped_operator := self.operators_map.get(operator.token_type):
168+
return f"{field_left} {mapped_operator} {field_right}"
169+
raise UnsupportedOperatorException(operator.token_type)
170+
171+
163172
class QueryRender(ABC):
164173
comment_symbol: str = None
165174
details: PlatformDetails = None
@@ -180,6 +189,13 @@ def render_not_supported_functions(self, not_supported_functions: list) -> str:
180189
not_supported_functions_str = "\n".join(line_template + func.lstrip() for func in not_supported_functions)
181190
return "\n\n" + self.wrap_with_comment(f"{self.unsupported_functions_text}\n{not_supported_functions_str}")
182191

192+
def wrap_with_not_supported_functions(self, query: str, not_supported_functions: Optional[list] = None) -> str:
193+
if not_supported_functions and wrap_query_with_meta_info_ctx_var.get():
194+
rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
195+
return query + rendered_not_supported
196+
197+
return query
198+
183199
def wrap_with_comment(self, value: str) -> str:
184200
return f"{self.comment_symbol} {value}"
185201

@@ -199,13 +215,14 @@ class PlatformQueryRender(QueryRender):
199215
group_token = "(%s)"
200216
query_parts_delimiter = " "
201217

202-
field_value_map = BaseQueryFieldValue(or_token=or_token)
218+
field_field_render = BaseFieldFieldRender()
219+
field_value_render = BaseFieldValueRender(or_token=or_token)
203220

204221
raw_log_field_pattern_map: ClassVar[dict[str, str]] = None
205222

206223
def __init__(self):
207224
super().__init__()
208-
self.operator_map = {
225+
self.logical_operators_map = {
209226
LogicalOperatorType.AND: f" {self.and_token} ",
210227
LogicalOperatorType.OR: f" {self.or_token} ",
211228
LogicalOperatorType.NOT: f" {self.not_token} ",
@@ -233,31 +250,34 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
233250

234251
def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
235252
if isinstance(token, FieldValue):
236-
if token.alias:
237-
field_name = token.alias.name
238-
else:
239-
mapped_fields = self.map_field(token.field, source_mapping)
240-
if len(mapped_fields) > 1:
241-
return self.group_token % self.operator_map[LogicalOperatorType.OR].join(
242-
[
243-
self.field_value_map.apply_field_value(
244-
field=field, operator=token.operator, value=token.value
245-
)
246-
for field in mapped_fields
247-
]
248-
)
249-
250-
field_name = mapped_fields[0]
251-
252-
return self.field_value_map.apply_field_value(field=field_name, operator=token.operator, value=token.value)
253-
253+
mapped_fields = [token.alias.name] if token.alias else self.map_field(token.field, source_mapping)
254+
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
255+
[
256+
self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value)
257+
for field in mapped_fields
258+
]
259+
)
260+
return self.group_token % joined if len(mapped_fields) > 1 else joined
261+
if isinstance(token, FieldField):
262+
alias_left, field_left = token.alias_left, token.field_left
263+
mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping)
264+
alias_right, field_right = token.alias_right, token.field_right
265+
mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping)
266+
cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right))
267+
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
268+
[
269+
self.field_field_render.apply_field_field(pair[0], token.operator, pair[1])
270+
for pair in cross_paired_fields
271+
]
272+
)
273+
return self.group_token % joined if len(cross_paired_fields) > 1 else joined
254274
if isinstance(token, Function):
255275
func_render = self.platform_functions.manager.get_in_query_render(token.name)
256276
return func_render.render(token, source_mapping)
257277
if isinstance(token, Keyword):
258-
return self.field_value_map.apply_field_value(field="", operator=token.operator, value=token.value)
278+
return self.field_value_render.apply_field_value(field="", operator=token.operator, value=token.value)
259279
if token.token_type in LogicalOperatorType:
260-
return self.operator_map.get(token.token_type)
280+
return self.logical_operators_map.get(token.token_type)
261281

262282
return token.token_type
263283

@@ -267,8 +287,8 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping
267287
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
268288
return "".join(result_values)
269289

270-
def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str:
271-
if meta_info and (meta_info.id or meta_info.title):
290+
def wrap_with_meta_info(self, query: str, meta_info: MetaInfoContainer) -> str:
291+
if wrap_query_with_meta_info_ctx_var.get() and meta_info and (meta_info.id or meta_info.title):
272292
meta_info_dict = {
273293
"name: ": meta_info.title,
274294
"uuid: ": meta_info.id,
@@ -301,11 +321,8 @@ def finalize_query(
301321
**kwargs, # noqa: ARG002
302322
) -> str:
303323
query = self._join_query_parts(prefix, query, functions)
304-
query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query)
305-
if not_supported_functions:
306-
rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
307-
return query + rendered_not_supported
308-
return query
324+
query = self.wrap_with_meta_info(query, meta_info)
325+
return self.wrap_with_not_supported_functions(query, not_supported_functions)
309326

310327
@staticmethod
311328
def unique_queries(queries_map: dict[str, str]) -> dict[str, dict[str]]:
@@ -336,7 +353,7 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp
336353

337354
return source_mappings
338355

339-
def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str:
356+
def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str:
340357
return self.finalize_query(
341358
prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info
342359
)
@@ -374,7 +391,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
374391
defined_raw_log_fields.append(prefix)
375392
return "\n".join(defined_raw_log_fields)
376393

377-
def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
394+
def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
378395
queries_map = {}
379396
errors = []
380397
source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids)
@@ -411,6 +428,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
411428

412429
def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str:
413430
if isinstance(query_container, RawQueryContainer):
414-
return self._generate_from_raw_query_container(query_container)
431+
return self.generate_from_raw_query_container(query_container)
415432

416-
return self._generate_from_tokenized_query_container(query_container)
433+
return self.generate_from_tokenized_query_container(query_container)

‎uncoder-core/app/translator/core/tokenizer.py

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from abc import ABC, abstractmethod
2121
from typing import Any, ClassVar, Optional, Union
2222

23+
from app.translator.core.const import TOKEN_TYPE
2324
from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType
2425
from app.translator.core.custom_types.values import ValueType
2526
from app.translator.core.escape_manager import EscapeManager
@@ -29,18 +30,18 @@
2930
UnsupportedOperatorException,
3031
)
3132
from app.translator.core.mapping import SourceMapping
32-
from app.translator.core.models.field import Field, FieldValue, Keyword
33+
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword
3334
from app.translator.core.models.functions.base import Function
3435
from app.translator.core.models.functions.eval import EvalArg
3536
from app.translator.core.models.functions.group_by import GroupByFunction
37+
from app.translator.core.models.functions.join import JoinFunction
3638
from app.translator.core.models.functions.rename import RenameArg
3739
from app.translator.core.models.functions.sort import SortArg
40+
from app.translator.core.models.functions.union import UnionFunction
3841
from app.translator.core.models.identifier import Identifier
3942
from app.translator.core.str_value_manager import StrValue, StrValueManager
4043
from app.translator.tools.utils import get_match_group
4144

42-
TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field]
43-
4445

4546
class BaseTokenizer(ABC):
4647
@abstractmethod
@@ -323,20 +324,31 @@ def filter_tokens(
323324
) -> list[TOKEN_TYPE]:
324325
return [token for token in tokens if isinstance(token, token_type)]
325326

326-
def get_field_tokens_from_func_args(
327+
def get_field_tokens_from_func_args( # noqa: PLR0912
327328
self, args: list[Union[Field, FieldValue, Keyword, Identifier, Function, SortArg]]
328329
) -> list[Field]:
329330
result = []
330331
for arg in args:
331332
if isinstance(arg, Field):
332333
result.append(arg)
334+
elif isinstance(arg, FieldField):
335+
if not arg.alias_left or arg.alias_left.name != arg.field_left.source_name:
336+
result.append(arg.field_left)
337+
if not arg.alias_right or arg.alias_right.name != arg.field_right.source_name:
338+
result.append(arg.field_right)
333339
elif isinstance(arg, FieldValue):
334340
if not arg.alias or arg.alias.name != arg.field.source_name:
335341
result.append(arg.field)
336342
elif isinstance(arg, GroupByFunction):
337343
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
338344
result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses))
339345
result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_]))
346+
elif isinstance(arg, (JoinFunction, UnionFunction)):
347+
result.extend(self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.tokens))
348+
result.extend(
349+
self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.functions.functions)
350+
)
351+
result.extend(self.get_field_tokens_from_func_args(args=arg.condition))
340352
elif isinstance(arg, Function):
341353
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
342354
elif isinstance(arg, SortArg) and isinstance(arg.field, Field):

‎uncoder-core/app/translator/platforms/athena/renders/athena.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@
2121
from app.translator.managers import render_manager
2222
from app.translator.platforms.athena.const import athena_details
2323
from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings
24-
from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender
24+
from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender
2525

2626

27-
class AthenaFieldValue(SqlFieldValue):
27+
class AthenaFieldValueRender(SqlFieldValueRender):
2828
details: PlatformDetails = athena_details
2929

3030

@@ -35,7 +35,7 @@ class AthenaQueryRender(SqlQueryRender):
3535

3636
or_token = "OR"
3737

38-
field_value_map = AthenaFieldValue(or_token=or_token)
38+
field_value_render = AthenaFieldValueRender(or_token=or_token)
3939
comment_symbol = "--"
4040
is_single_line_comment = True
4141

‎uncoder-core/app/translator/platforms/base/aql/renders/aql.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121

2222
from app.translator.const import DEFAULT_VALUE_TYPE
2323
from app.translator.core.custom_types.values import ValueType
24-
from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender
24+
from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
2525
from app.translator.core.str_value_manager import StrValue
2626
from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings
2727
from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager
2828

2929

30-
class AQLFieldValue(BaseQueryFieldValue):
30+
class AQLFieldValueRender(BaseFieldValueRender):
3131
str_value_manager = aql_str_value_manager
3232

3333
@staticmethod
@@ -127,8 +127,6 @@ class AQLQueryRender(PlatformQueryRender):
127127
and_token = "AND"
128128
not_token = "NOT"
129129

130-
field_value_map = AQLFieldValue(or_token=or_token)
131-
132130
def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002
133131
table = str(log_source_signature)
134132
extra_condition = log_source_signature.extra_condition

‎uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121

2222
from app.translator.const import DEFAULT_VALUE_TYPE
2323
from app.translator.core.custom_types.values import ValueType
24-
from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender
24+
from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender
2525
from app.translator.core.str_value_manager import StrValue
2626
from app.translator.platforms.base.lucene.mapping import LuceneLogSourceSignature
2727
from app.translator.platforms.base.lucene.str_value_manager import lucene_str_value_manager
2828

2929

30-
class LuceneFieldValue(BaseQueryFieldValue):
30+
class LuceneFieldValueRender(BaseFieldValueRender):
3131
str_value_manager = lucene_str_value_manager
3232

3333
@staticmethod

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy