Skip to content

Commit 0c9b2a0

Browse files
alexvolhaalexvolha
authored
Merge pull request #157 from UncoderIO/gis-7997
Gis 7997
2 parents 68d3e0e + 594c661 commit 0c9b2a0

File tree

40 files changed

+258
-187
lines changed

40 files changed

+258
-187
lines changed

‎uncoder-core/app/translator/core/const.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
from typing import Union
2+
3+
from app.translator.core.models.field import Alias, Field, FieldValue, Keyword
4+
from app.translator.core.models.identifier import Identifier
5+
6+
TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias]

‎uncoder-core/app/translator/core/context_vars.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,10 @@
11
from contextvars import ContextVar
2+
from typing import Optional
23

34
return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False)
45
"""Set to True to return only first query if rendered multiple options"""
6+
7+
wrap_query_with_meta_info_ctx_var: ContextVar[bool] = ContextVar("wrap_query_with_meta_info_ctx_var", default=True)
8+
"""Set to False not to wrap query with meta info commentary"""
9+
10+
preset_log_source_str_ctx_var: ContextVar[Optional[str]] = ContextVar("preset_log_source_str_ctx_var", default=None)

‎uncoder-core/app/translator/core/custom_types/functions.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ class FunctionType(CustomEnum):
2828
bin = "bin"
2929
eval = "eval"
3030
fields = "fields"
31+
join = "join"
3132
rename = "rename"
3233
search = "search"
3334
sort_limit = "sort_limit"

‎uncoder-core/app/translator/core/models/field.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,22 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma
3737
self.__generic_names_map = generic_names_map
3838

3939

40+
class FieldField:
41+
def __init__(
42+
self,
43+
source_name_left: str,
44+
operator: Identifier,
45+
source_name_right: str,
46+
is_alias_left: bool = False,
47+
is_alias_right: bool = False,
48+
):
49+
self.field_left = Field(source_name=source_name_left)
50+
self.alias_left = Alias(name=source_name_left) if is_alias_left else None
51+
self.operator = operator
52+
self.field_right = Field(source_name=source_name_right)
53+
self.alias_right = Alias(name=source_name_right) if is_alias_right else None
54+
55+
4056
class FieldValue:
4157
def __init__(
4258
self,

‎uncoder-core/app/translator/core/models/functions/join.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
from dataclasses import dataclass, field
2+
from typing import Union
3+
4+
from app.translator.core.custom_types.functions import FunctionType
5+
from app.translator.core.models.field import Alias, Field
6+
from app.translator.core.models.functions.base import Function
7+
from app.translator.core.models.identifier import Identifier
8+
from app.translator.core.models.query_container import TokenizedQueryContainer
9+
from app.translator.tools.custom_enum import CustomEnum
10+
11+
12+
class JoinType(CustomEnum):
13+
inner = "inner"
14+
left = "left"
15+
right = "right"
16+
cross = "cross"
17+
18+
19+
@dataclass
20+
class JoinFunction(Function):
21+
name: str = FunctionType.join
22+
alias: Alias = None
23+
type_: str = JoinType.inner
24+
tokenized_query_container: TokenizedQueryContainer = None
25+
condition: list[Union[Alias, Field, Identifier]] = field(default_factory=list)
26+
preset_log_source_str: str = None

‎uncoder-core/app/translator/core/models/functions/union.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from dataclasses import dataclass
2+
3+
from app.translator.core.custom_types.functions import FunctionType
4+
from app.translator.core.models.functions.base import Function
5+
from app.translator.core.models.query_container import TokenizedQueryContainer
6+
7+
8+
@dataclass
9+
class UnionFunction(Function):
10+
name: str = FunctionType.union
11+
tokenized_query_container: TokenizedQueryContainer = None
12+
preset_log_source_str: str = None

‎uncoder-core/app/translator/core/models/query_container.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
from datetime import datetime
44
from typing import Optional
55

6+
from app.translator.core.const import TOKEN_TYPE
67
from app.translator.core.custom_types.meta_info import SeverityType
78
from app.translator.core.mapping import DEFAULT_MAPPING_NAME
89
from app.translator.core.models.field import Field
910
from app.translator.core.models.functions.base import ParsedFunctions
10-
from app.translator.core.tokenizer import TOKEN_TYPE
1111

1212

1313
class MetaInfoContainer:

‎uncoder-core/app/translator/core/parser.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from abc import ABC, abstractmethod
2121
from typing import Union
2222

23+
from app.translator.core.const import TOKEN_TYPE
2324
from app.translator.core.exceptions.parser import TokenizerGeneralException
2425
from app.translator.core.functions import PlatformFunctions
2526
from app.translator.core.mapping import BasePlatformMappings, SourceMapping
@@ -28,7 +29,7 @@
2829
from app.translator.core.models.identifier import Identifier
2930
from app.translator.core.models.platform_details import PlatformDetails
3031
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
31-
from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer
32+
from app.translator.core.tokenizer import QueryTokenizer
3233

3334

3435
class QueryParser(ABC):

‎uncoder-core/app/translator/core/render.py

Lines changed: 57 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -16,36 +16,36 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19-
19+
import itertools
2020
from abc import ABC, abstractmethod
2121
from collections.abc import Callable
2222
from typing import ClassVar, Optional, Union
2323

2424
from app.translator.const import DEFAULT_VALUE_TYPE
25-
from app.translator.core.context_vars import return_only_first_query_ctx_var
25+
from app.translator.core.const import TOKEN_TYPE
26+
from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var
2627
from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType
2728
from app.translator.core.custom_types.values import ValueType
2829
from app.translator.core.escape_manager import EscapeManager
2930
from app.translator.core.exceptions.core import NotImplementedException, StrictPlatformException
3031
from app.translator.core.exceptions.parser import UnsupportedOperatorException
3132
from app.translator.core.functions import PlatformFunctions
3233
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
33-
from app.translator.core.models.field import Field, FieldValue, Keyword
34+
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword
3435
from app.translator.core.models.functions.base import Function, RenderedFunctions
3536
from app.translator.core.models.identifier import Identifier
3637
from app.translator.core.models.platform_details import PlatformDetails
3738
from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer
3839
from app.translator.core.str_value_manager import StrValue, StrValueManager
39-
from app.translator.core.tokenizer import TOKEN_TYPE
4040

4141

42-
class BaseQueryFieldValue(ABC):
42+
class BaseFieldValueRender(ABC):
4343
details: PlatformDetails = None
4444
escape_manager: EscapeManager = None
4545
str_value_manager: StrValueManager = None
4646

4747
def __init__(self, or_token: str):
48-
self.field_value: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = {
48+
self.modifiers_map: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = {
4949
OperatorType.EQ: self.equal_modifier,
5050
OperatorType.NOT_EQ: self.not_equal_modifier,
5151
OperatorType.LT: self.less_modifier,
@@ -155,11 +155,20 @@ def apply_value(self, value: Union[str, int], value_type: str = ValueType.value)
155155
return self.escape_manager.escape(value, value_type)
156156

157157
def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VALUE_TYPE) -> str:
158-
if modifier_function := self.field_value.get(operator.token_type):
158+
if modifier_function := self.modifiers_map.get(operator.token_type):
159159
return modifier_function(field, value)
160160
raise UnsupportedOperatorException(operator.token_type)
161161

162162

163+
class BaseFieldFieldRender(ABC):
164+
operators_map: ClassVar[dict[str, str]] = {}
165+
166+
def apply_field_field(self, field_left: str, operator: Identifier, field_right: str) -> str:
167+
if mapped_operator := self.operators_map.get(operator.token_type):
168+
return f"{field_left} {mapped_operator} {field_right}"
169+
raise UnsupportedOperatorException(operator.token_type)
170+
171+
163172
class QueryRender(ABC):
164173
comment_symbol: str = None
165174
details: PlatformDetails = None
@@ -180,6 +189,13 @@ def render_not_supported_functions(self, not_supported_functions: list) -> str:
180189
not_supported_functions_str = "\n".join(line_template + func.lstrip() for func in not_supported_functions)
181190
return "\n\n" + self.wrap_with_comment(f"{self.unsupported_functions_text}\n{not_supported_functions_str}")
182191

192+
def wrap_with_not_supported_functions(self, query: str, not_supported_functions: Optional[list] = None) -> str:
193+
if not_supported_functions and wrap_query_with_meta_info_ctx_var.get():
194+
rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
195+
return query + rendered_not_supported
196+
197+
return query
198+
183199
def wrap_with_comment(self, value: str) -> str:
184200
return f"{self.comment_symbol} {value}"
185201

@@ -199,13 +215,14 @@ class PlatformQueryRender(QueryRender):
199215
group_token = "(%s)"
200216
query_parts_delimiter = " "
201217

202-
field_value_map = BaseQueryFieldValue(or_token=or_token)
218+
field_field_render = BaseFieldFieldRender()
219+
field_value_render = BaseFieldValueRender(or_token=or_token)
203220

204221
raw_log_field_pattern_map: ClassVar[dict[str, str]] = None
205222

206223
def __init__(self):
207224
super().__init__()
208-
self.operator_map = {
225+
self.logical_operators_map = {
209226
LogicalOperatorType.AND: f" {self.and_token} ",
210227
LogicalOperatorType.OR: f" {self.or_token} ",
211228
LogicalOperatorType.NOT: f" {self.not_token} ",
@@ -233,31 +250,34 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]:
233250

234251
def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
235252
if isinstance(token, FieldValue):
236-
if token.alias:
237-
field_name = token.alias.name
238-
else:
239-
mapped_fields = self.map_field(token.field, source_mapping)
240-
if len(mapped_fields) > 1:
241-
return self.group_token % self.operator_map[LogicalOperatorType.OR].join(
242-
[
243-
self.field_value_map.apply_field_value(
244-
field=field, operator=token.operator, value=token.value
245-
)
246-
for field in mapped_fields
247-
]
248-
)
249-
250-
field_name = mapped_fields[0]
251-
252-
return self.field_value_map.apply_field_value(field=field_name, operator=token.operator, value=token.value)
253-
253+
mapped_fields = [token.alias.name] if token.alias else self.map_field(token.field, source_mapping)
254+
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
255+
[
256+
self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value)
257+
for field in mapped_fields
258+
]
259+
)
260+
return self.group_token % joined if len(mapped_fields) > 1 else joined
261+
if isinstance(token, FieldField):
262+
alias_left, field_left = token.alias_left, token.field_left
263+
mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping)
264+
alias_right, field_right = token.alias_right, token.field_right
265+
mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping)
266+
cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right))
267+
joined = self.logical_operators_map[LogicalOperatorType.OR].join(
268+
[
269+
self.field_field_render.apply_field_field(pair[0], token.operator, pair[1])
270+
for pair in cross_paired_fields
271+
]
272+
)
273+
return self.group_token % joined if len(cross_paired_fields) > 1 else joined
254274
if isinstance(token, Function):
255275
func_render = self.platform_functions.manager.get_in_query_render(token.name)
256276
return func_render.render(token, source_mapping)
257277
if isinstance(token, Keyword):
258-
return self.field_value_map.apply_field_value(field="", operator=token.operator, value=token.value)
278+
return self.field_value_render.apply_field_value(field="", operator=token.operator, value=token.value)
259279
if token.token_type in LogicalOperatorType:
260-
return self.operator_map.get(token.token_type)
280+
return self.logical_operators_map.get(token.token_type)
261281

262282
return token.token_type
263283

@@ -273,8 +293,8 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping
273293
raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields))
274294
return "".join(result_values)
275295

276-
def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str:
277-
if meta_info and (meta_info.id or meta_info.title):
296+
def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str:
297+
if wrap_query_with_meta_info_ctx_var.get() and meta_info and (meta_info.id or meta_info.title):
278298
meta_info_dict = {
279299
"name: ": meta_info.title,
280300
"uuid: ": meta_info.id,
@@ -307,11 +327,8 @@ def finalize_query(
307327
**kwargs, # noqa: ARG002
308328
) -> str:
309329
query = self._join_query_parts(prefix, query, functions)
310-
query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query)
311-
if not_supported_functions:
312-
rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
313-
return query + rendered_not_supported
314-
return query
330+
query = self.wrap_with_meta_info(query, meta_info)
331+
return self.wrap_with_not_supported_functions(query, not_supported_functions)
315332

316333
@staticmethod
317334
def unique_queries(queries_map: dict[str, str]) -> dict[str, dict[str]]:
@@ -342,7 +359,7 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp
342359

343360
return source_mappings
344361

345-
def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str:
362+
def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str:
346363
return self.finalize_query(
347364
prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info
348365
)
@@ -380,7 +397,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
380397
defined_raw_log_fields.append(prefix)
381398
return "\n".join(defined_raw_log_fields)
382399

383-
def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
400+
def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
384401
queries_map = {}
385402
errors = []
386403
source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids)
@@ -417,6 +434,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
417434

418435
def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str:
419436
if isinstance(query_container, RawQueryContainer):
420-
return self._generate_from_raw_query_container(query_container)
437+
return self.generate_from_raw_query_container(query_container)
421438

422-
return self._generate_from_tokenized_query_container(query_container)
439+
return self.generate_from_tokenized_query_container(query_container)

‎uncoder-core/app/translator/core/tokenizer.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from abc import ABC, abstractmethod
2121
from typing import Any, ClassVar, Optional, Union
2222

23+
from app.translator.core.const import TOKEN_TYPE
2324
from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType
2425
from app.translator.core.custom_types.values import ValueType
2526
from app.translator.core.escape_manager import EscapeManager
@@ -29,18 +30,18 @@
2930
UnsupportedOperatorException,
3031
)
3132
from app.translator.core.mapping import SourceMapping
32-
from app.translator.core.models.field import Field, FieldValue, Keyword
33+
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword
3334
from app.translator.core.models.functions.base import Function
3435
from app.translator.core.models.functions.eval import EvalArg
3536
from app.translator.core.models.functions.group_by import GroupByFunction
37+
from app.translator.core.models.functions.join import JoinFunction
3638
from app.translator.core.models.functions.rename import RenameArg
3739
from app.translator.core.models.functions.sort import SortArg
40+
from app.translator.core.models.functions.union import UnionFunction
3841
from app.translator.core.models.identifier import Identifier
3942
from app.translator.core.str_value_manager import StrValue, StrValueManager
4043
from app.translator.tools.utils import get_match_group
4144

42-
TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field]
43-
4445

4546
class BaseTokenizer(ABC):
4647
@abstractmethod
@@ -323,20 +324,27 @@ def filter_tokens(
323324
) -> list[TOKEN_TYPE]:
324325
return [token for token in tokens if isinstance(token, token_type)]
325326

326-
def get_field_tokens_from_func_args(
327+
def get_field_tokens_from_func_args( # noqa: PLR0912
327328
self, args: list[Union[Field, FieldValue, Keyword, Identifier, Function, SortArg]]
328329
) -> list[Field]:
329330
result = []
330331
for arg in args:
331332
if isinstance(arg, Field):
332333
result.append(arg)
334+
elif isinstance(arg, FieldField):
335+
if not arg.alias_left or arg.alias_left.name != arg.field_left.source_name:
336+
result.append(arg.field_left)
337+
if not arg.alias_right or arg.alias_right.name != arg.field_right.source_name:
338+
result.append(arg.field_right)
333339
elif isinstance(arg, FieldValue):
334340
if not arg.alias or arg.alias.name != arg.field.source_name:
335341
result.append(arg.field)
336342
elif isinstance(arg, GroupByFunction):
337343
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
338344
result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses))
339345
result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_]))
346+
elif isinstance(arg, (JoinFunction, UnionFunction)):
347+
continue
340348
elif isinstance(arg, Function):
341349
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
342350
elif isinstance(arg, SortArg) and isinstance(arg.field, Field):

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy