From b40132aa1ba802652aa23bafa55d0ff8a6501381 Mon Sep 17 00:00:00 2001 From: Nazar Gesyk Date: Wed, 17 Jul 2024 11:29:12 +0300 Subject: [PATCH 1/2] Improve filter_tokens in core tokenizer --- uncoder-core/app/translator/core/tokenizer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index 08295917..a9828115 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -338,7 +338,7 @@ def tokenize(self, query: str) -> list[QUERY_TOKEN_TYPE]: @staticmethod def filter_tokens( tokens: list[QUERY_TOKEN_TYPE], - token_type: Union[type[FieldValue], type[Field], type[Keyword], type[Identifier]], + token_type: Union[type[FieldValue], type[Field], type[FieldField], type[Keyword], type[Identifier]], ) -> list[QUERY_TOKEN_TYPE]: return [token for token in tokens if isinstance(token, token_type)] @@ -363,7 +363,9 @@ def get_field_tokens_from_func_args( # noqa: PLR0912 result.extend(self.get_field_tokens_from_func_args(args=arg.args)) result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_])) - elif isinstance(arg, (JoinFunction, UnionFunction)): + elif isinstance(arg, JoinFunction): + result.extend(self.get_field_tokens_from_func_args(args=arg.condition)) + elif isinstance(arg, UnionFunction): continue elif isinstance(arg, Function): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) From 90da0d735343f6a8c7bb76c3db4702be519e0c37 Mon Sep 17 00:00:00 2001 From: Nazar Gesyk Date: Wed, 17 Jul 2024 11:50:35 +0300 Subject: [PATCH 2/2] Improve get_field_tokens in core parser --- uncoder-core/app/translator/core/parser.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index c51ada8c..fcefeb69 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -28,6 +28,7 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.models.query_tokens.field import Field +from app.translator.core.models.query_tokens.field_field import FieldField from app.translator.core.models.query_tokens.field_value import FieldValue from app.translator.core.models.query_tokens.function_value import FunctionValue from app.translator.core.tokenizer import QueryTokenizer @@ -68,6 +69,11 @@ def get_field_tokens( for token in query_tokens: if isinstance(token, FieldValue): field_tokens.append(token.field) + elif isinstance(token, FieldField): + if token.field_left: + field_tokens.append(token.field_left) + if token.field_right: + field_tokens.append(token.field_right) elif isinstance(token, FunctionValue): field_tokens.extend(self.tokenizer.get_field_tokens_from_func_args([token.function])) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy