Skip to content

Commit 1cf80f3

Browse files
alexvolhaalexvolha
committed
function value
1 parent 4b54f66 commit 1cf80f3

File tree

15 files changed

+245
-128
lines changed

15 files changed

+245
-128
lines changed

‎uncoder-core/app/translator/core/const.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from typing import Union
22

33
from app.translator.core.models.field import Alias, Field, FieldValue, Keyword
4+
from app.translator.core.models.function_value import FunctionValue
45
from app.translator.core.models.identifier import Identifier
56

6-
TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias]
7+
QUERY_TOKEN_TYPE = Union[FieldValue, FunctionValue, Keyword, Identifier, Field, Alias]

‎uncoder-core/app/translator/core/models/function_value.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
from typing import Optional, Union
2+
3+
from app.translator.core.custom_types.tokens import STR_SEARCH_OPERATORS
4+
from app.translator.core.models.functions.base import Function
5+
from app.translator.core.models.identifier import Identifier
6+
from app.translator.core.str_value_manager import StrValue
7+
8+
9+
class FunctionValue:
10+
def __init__(self, function: Function, operator: Identifier, value: Union[int, str, StrValue, list, tuple]):
11+
self.function = function
12+
self.operator = operator
13+
self.values = []
14+
self.__add_value(value)
15+
16+
@property
17+
def value(self) -> Union[int, str, StrValue, list[Union[int, str, StrValue]]]:
18+
if isinstance(self.values, list) and len(self.values) == 1:
19+
return self.values[0]
20+
return self.values
21+
22+
@value.setter
23+
def value(self, new_value: Union[int, str, StrValue, list[Union[int, str, StrValue]]]) -> None:
24+
self.values = []
25+
self.__add_value(new_value)
26+
27+
def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -> None:
28+
if value and isinstance(value, (list, tuple)):
29+
for v in value:
30+
self.__add_value(v)
31+
elif (
32+
value
33+
and isinstance(value, str)
34+
and value.isnumeric()
35+
and self.operator.token_type not in STR_SEARCH_OPERATORS
36+
):
37+
self.values.append(int(value))
38+
elif value is not None and isinstance(value, (int, str)):
39+
self.values.append(value)

‎uncoder-core/app/translator/core/models/query_container.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from datetime import datetime
44
from typing import Optional
55

6-
from app.translator.core.const import TOKEN_TYPE
6+
from app.translator.core.const import QUERY_TOKEN_TYPE
77
from app.translator.core.custom_types.meta_info import SeverityType
88
from app.translator.core.mapping import DEFAULT_MAPPING_NAME
99
from app.translator.core.models.field import Field
@@ -65,6 +65,6 @@ class RawQueryDictContainer:
6565

6666
@dataclass
6767
class TokenizedQueryContainer:
68-
tokens: list[TOKEN_TYPE]
68+
tokens: list[QUERY_TOKEN_TYPE]
6969
meta_info: MetaInfoContainer
7070
functions: ParsedFunctions = field(default_factory=ParsedFunctions)

‎uncoder-core/app/translator/core/parser.py

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@
1818

1919
import re
2020
from abc import ABC, abstractmethod
21-
from typing import Union
21+
from typing import Optional, Union
2222

23-
from app.translator.core.const import TOKEN_TYPE
23+
from app.translator.core.const import QUERY_TOKEN_TYPE
2424
from app.translator.core.exceptions.parser import TokenizerGeneralException
2525
from app.translator.core.functions import PlatformFunctions
2626
from app.translator.core.mapping import BasePlatformMappings, SourceMapping
27-
from app.translator.core.models.field import Field, FieldValue, Keyword
28-
from app.translator.core.models.functions.base import ParsedFunctions
29-
from app.translator.core.models.identifier import Identifier
27+
from app.translator.core.models.field import Field, FieldValue
28+
from app.translator.core.models.function_value import FunctionValue
29+
from app.translator.core.models.functions.base import Function
3030
from app.translator.core.models.platform_details import PlatformDetails
3131
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
3232
from app.translator.core.tokenizer import QueryTokenizer
@@ -55,24 +55,30 @@ class PlatformQueryParser(QueryParser, ABC):
5555
tokenizer: QueryTokenizer = None
5656
platform_functions: PlatformFunctions = None
5757

58-
def get_fields_tokens(self, tokens: list[Union[FieldValue, Keyword, Identifier]]) -> list[Field]:
59-
return [token.field for token in self.tokenizer.filter_tokens(tokens, FieldValue)]
60-
61-
def get_tokens_and_source_mappings(
62-
self, query: str, log_sources: dict[str, Union[str, list[str]]]
63-
) -> tuple[list[TOKEN_TYPE], list[SourceMapping]]:
58+
def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]:
6459
if not query:
6560
raise TokenizerGeneralException("Can't translate empty query. Please provide more details")
66-
tokens = self.tokenizer.tokenize(query=query)
67-
field_tokens = self.get_fields_tokens(tokens=tokens)
61+
return self.tokenizer.tokenize(query=query)
62+
63+
def get_field_tokens(
64+
self, query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None
65+
) -> list[Field]:
66+
field_tokens = []
67+
for token in query_tokens:
68+
if isinstance(token, FieldValue):
69+
field_tokens.append(token.field)
70+
elif isinstance(token, FunctionValue):
71+
field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args([token.function]))
72+
73+
if functions:
74+
field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args(functions))
75+
76+
return field_tokens
77+
78+
def get_source_mappings(
79+
self, field_tokens: list[Field], log_sources: dict[str, Union[str, list[str]]]
80+
) -> list[SourceMapping]:
6881
field_names = [field.source_name for field in field_tokens]
6982
source_mappings = self.mappings.get_suitable_source_mappings(field_names=field_names, **log_sources)
7083
self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping)
71-
72-
return tokens, source_mappings
73-
74-
def set_functions_fields_generic_names(
75-
self, functions: ParsedFunctions, source_mappings: list[SourceMapping]
76-
) -> None:
77-
field_tokens = self.tokenizer.get_field_tokens_from_func_args(args=functions.functions)
78-
self.tokenizer.set_field_tokens_generic_names_map(field_tokens, source_mappings, self.mappings.default_mapping)
84+
return source_mappings

‎uncoder-core/app/translator/core/render.py

Lines changed: 40 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
from typing import ClassVar, Optional, Union
2323

2424
from app.translator.const import DEFAULT_VALUE_TYPE
25-
from app.translator.core.const import TOKEN_TYPE
25+
from app.translator.core.const import QUERY_TOKEN_TYPE
2626
from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var
2727
from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType
2828
from app.translator.core.custom_types.values import ValueType
@@ -32,6 +32,7 @@
3232
from app.translator.core.functions import PlatformFunctions
3333
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
3434
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword, PredefinedField
35+
from app.translator.core.models.function_value import FunctionValue
3536
from app.translator.core.models.functions.base import Function, RenderedFunctions
3637
from app.translator.core.models.identifier import Identifier
3738
from app.translator.core.models.platform_details import PlatformDetails
@@ -258,7 +259,9 @@ def map_predefined_field(self, predefined_field: PredefinedField) -> str:
258259

259260
return mapped_predefined_field_name
260261

261-
def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str:
262+
def apply_token( # noqa: PLR0911
263+
self, token: Union[FieldValue, Function, Keyword, Identifier], source_mapping: SourceMapping
264+
) -> str:
262265
if isinstance(token, FieldValue):
263266
if token.alias:
264267
mapped_fields = [token.alias.name]
@@ -286,6 +289,12 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
286289
]
287290
)
288291
return self.group_token % joined if len(cross_paired_fields) > 1 else joined
292+
if isinstance(token, FunctionValue):
293+
func_render = self.platform_functions.manager.get_in_query_render(token.function.name)
294+
rendered_func = func_render.render(token.function, source_mapping)
295+
return self.field_value_render.apply_field_value(
296+
field=rendered_func, operator=token.operator, value=token.value
297+
)
289298
if isinstance(token, Function):
290299
func_render = self.platform_functions.manager.get_in_query_render(token.name)
291300
return func_render.render(token, source_mapping)
@@ -296,7 +305,7 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
296305

297306
return token.token_type
298307

299-
def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str:
308+
def generate_query(self, tokens: list[QUERY_TOKEN_TYPE], source_mapping: SourceMapping) -> str:
300309
result_values = []
301310
unmapped_fields = set()
302311
for token in tokens:
@@ -412,37 +421,45 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
412421
defined_raw_log_fields.append(prefix)
413422
return "\n".join(defined_raw_log_fields)
414423

424+
def _generate_from_tokenized_query_container_by_source_mapping(
425+
self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping
426+
) -> str:
427+
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
428+
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)
429+
430+
if source_mapping.raw_log_fields:
431+
defined_raw_log_fields = self.generate_raw_log_fields(
432+
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
433+
)
434+
prefix += f"\n{defined_raw_log_fields}"
435+
query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
436+
not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported
437+
return self.finalize_query(
438+
prefix=prefix,
439+
query=query,
440+
functions=rendered_functions.rendered,
441+
not_supported_functions=not_supported_functions,
442+
meta_info=query_container.meta_info,
443+
source_mapping=source_mapping,
444+
)
445+
415446
def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
416447
queries_map = {}
417448
errors = []
418449
source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids)
419450

420451
for source_mapping in source_mappings:
421-
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
422-
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)
423452
try:
424-
if source_mapping.raw_log_fields:
425-
defined_raw_log_fields = self.generate_raw_log_fields(
426-
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
427-
)
428-
prefix += f"\n{defined_raw_log_fields}"
429-
result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
430-
except StrictPlatformException as err:
431-
errors.append(err)
432-
continue
433-
else:
434-
not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported
435-
finalized_query = self.finalize_query(
436-
prefix=prefix,
437-
query=result,
438-
functions=rendered_functions.rendered,
439-
not_supported_functions=not_supported_functions,
440-
meta_info=query_container.meta_info,
441-
source_mapping=source_mapping,
453+
finalized_query = self._generate_from_tokenized_query_container_by_source_mapping(
454+
query_container, source_mapping
442455
)
443456
if return_only_first_query_ctx_var.get() is True:
444457
return finalized_query
445458
queries_map[source_mapping.source_id] = finalized_query
459+
except StrictPlatformException as err:
460+
errors.append(err)
461+
continue
462+
446463
if not queries_map and errors:
447464
raise errors[0]
448465
return self.finalize(queries_map)

‎uncoder-core/app/translator/core/tokenizer.py

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,20 @@
2020
from abc import ABC, abstractmethod
2121
from typing import Any, ClassVar, Optional, Union
2222

23-
from app.translator.core.const import TOKEN_TYPE
23+
from app.translator.core.const import QUERY_TOKEN_TYPE
2424
from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType
2525
from app.translator.core.custom_types.values import ValueType
2626
from app.translator.core.escape_manager import EscapeManager
27+
from app.translator.core.exceptions.functions import NotSupportedFunctionException
2728
from app.translator.core.exceptions.parser import (
2829
QueryParenthesesException,
2930
TokenizerGeneralException,
3031
UnsupportedOperatorException,
3132
)
33+
from app.translator.core.functions import PlatformFunctions
3234
from app.translator.core.mapping import SourceMapping
3335
from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword
36+
from app.translator.core.models.function_value import FunctionValue
3437
from app.translator.core.models.functions.base import Function
3538
from app.translator.core.models.functions.eval import EvalArg
3639
from app.translator.core.models.functions.group_by import GroupByFunction
@@ -64,6 +67,7 @@ class QueryTokenizer(BaseTokenizer):
6467

6568
# do not modify, use subclasses to define this attribute
6669
field_pattern: str = None
70+
function_pattern: str = None
6771
_value_pattern: str = None
6872
value_pattern: str = None
6973
multi_value_pattern: str = None
@@ -73,6 +77,7 @@ class QueryTokenizer(BaseTokenizer):
7377
wildcard_symbol = None
7478
escape_manager: EscapeManager = None
7579
str_value_manager: StrValueManager = None
80+
platform_functions: PlatformFunctions = None
7681

7782
def __init_subclass__(cls, **kwargs):
7883
cls._validate_re_patterns()
@@ -268,9 +273,16 @@ def _check_field_value_match(self, query: str, white_space_pattern: str = r"\s+"
268273

269274
return False
270275

276+
def search_function_value(self, query: str) -> tuple[FunctionValue, str]: # noqa: ARG002
277+
raise NotSupportedFunctionException
278+
279+
@staticmethod
280+
def _check_function_value_match(query: str) -> bool: # noqa: ARG004
281+
return False
282+
271283
def _get_next_token(
272284
self, query: str
273-
) -> tuple[Union[FieldValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]:
285+
) -> tuple[Union[FieldValue, FunctionValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]:
274286
query = query.strip("\n").strip(" ").strip("\n")
275287
if query.startswith(GroupType.L_PAREN):
276288
return Identifier(token_type=GroupType.L_PAREN), query[1:]
@@ -280,6 +292,8 @@ def _get_next_token(
280292
logical_operator = logical_operator_search.group("logical_operator")
281293
pos = logical_operator_search.end()
282294
return Identifier(token_type=logical_operator.lower()), query[pos:]
295+
if self.platform_functions and self._check_function_value_match(query):
296+
return self.search_function_value(query)
283297
if self._check_field_value_match(query):
284298
return self.search_field_value(query)
285299
if self.keyword_pattern and re.match(self.keyword_pattern, query):
@@ -288,7 +302,7 @@ def _get_next_token(
288302
raise TokenizerGeneralException("Unsupported query entry")
289303

290304
@staticmethod
291-
def _validate_parentheses(tokens: list[TOKEN_TYPE]) -> None:
305+
def _validate_parentheses(tokens: list[QUERY_TOKEN_TYPE]) -> None:
292306
parentheses = []
293307
for token in tokens:
294308
if isinstance(token, Identifier) and token.token_type in (GroupType.L_PAREN, GroupType.R_PAREN):
@@ -320,8 +334,9 @@ def tokenize(self, query: str) -> list[Union[FieldValue, Keyword, Identifier]]:
320334

321335
@staticmethod
322336
def filter_tokens(
323-
tokens: list[TOKEN_TYPE], token_type: Union[type[FieldValue], type[Field], type[Keyword], type[Identifier]]
324-
) -> list[TOKEN_TYPE]:
337+
tokens: list[QUERY_TOKEN_TYPE],
338+
token_type: Union[type[FieldValue], type[Field], type[Keyword], type[Identifier]],
339+
) -> list[QUERY_TOKEN_TYPE]:
325340
return [token for token in tokens if isinstance(token, token_type)]
326341

327342
def get_field_tokens_from_func_args( # noqa: PLR0912
@@ -339,6 +354,8 @@ def get_field_tokens_from_func_args( # noqa: PLR0912
339354
elif isinstance(arg, FieldValue):
340355
if arg.field:
341356
result.append(arg.field)
357+
elif isinstance(arg, FunctionValue):
358+
result.extend(self.get_field_tokens_from_func_args(args=[arg.function]))
342359
elif isinstance(arg, GroupByFunction):
343360
result.extend(self.get_field_tokens_from_func_args(args=arg.args))
344361
result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses))

‎uncoder-core/app/translator/platforms/base/aql/parsers/aql.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@
2727
from app.translator.platforms.base.aql.functions import AQLFunctions, aql_functions
2828
from app.translator.platforms.base.aql.log_source_map import LOG_SOURCE_FUNCTIONS_MAP
2929
from app.translator.platforms.base.aql.mapping import AQLMappings, aql_mappings
30-
from app.translator.platforms.base.aql.tokenizer import AQLTokenizer, aql_tokenizer
30+
from app.translator.platforms.base.aql.tokenizer import AQLTokenizer
3131
from app.translator.tools.utils import get_match_group
3232

3333

3434
class AQLQueryParser(PlatformQueryParser):
35-
tokenizer: AQLTokenizer = aql_tokenizer
35+
tokenizer: AQLTokenizer = AQLTokenizer(aql_functions)
3636
mappings: AQLMappings = aql_mappings
3737
platform_functions: AQLFunctions = aql_functions
3838

@@ -116,10 +116,10 @@ def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[
116116

117117
def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer:
118118
query, log_sources, functions = self._parse_query(raw_query_container.query)
119-
tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources)
120-
fields_tokens = self.get_fields_tokens(tokens=tokens)
121-
self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings)
119+
query_tokens = self.get_query_tokens(query)
120+
field_tokens = self.get_field_tokens(query_tokens, functions.functions)
121+
source_mappings = self.get_source_mappings(field_tokens, log_sources)
122122
meta_info = raw_query_container.meta_info
123-
meta_info.query_fields = fields_tokens
123+
meta_info.query_fields = field_tokens
124124
meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings]
125-
return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions)
125+
return TokenizedQueryContainer(tokens=query_tokens, meta_info=meta_info, functions=functions)

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy