parse regex

alexvolha · alexvolha · commit dcf1125f213d · 2024-10-18T10:59:49.000+03:00
diff --git a/uncoder-core/app/translator/platforms/base/sql/tokenizer.py b/uncoder-core/app/translator/platforms/base/sql/tokenizer.py
@@ -22,7 +22,9 @@
 from app.translator.core.custom_types.tokens import OperatorType
 from app.translator.core.custom_types.values import ValueType
 from app.translator.core.models.query_tokens.field_value import FieldValue
+from app.translator.core.models.query_tokens.function_value import FunctionValue
 from app.translator.core.models.query_tokens.identifier import Identifier
+from app.translator.core.models.query_tokens.keyword import Keyword
 from app.translator.core.tokenizer import QueryTokenizer
 from app.translator.platforms.base.sql.str_value_manager import sql_str_value_manager
 from app.translator.tools.utils import get_match_group
@@ -49,6 +51,7 @@ class SqlTokenizer(QueryTokenizer):
     )
     _value_pattern = rf"{num_value_pattern}|{bool_value_pattern}|{single_quotes_value_pattern}"
     multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>\d+(?:,\s*\d+)*|'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*'(?:,\s*'(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{{\}}\s]|'')*')*)\)"""  # noqa: E501
+    re_field_value_pattern = rf"""regexp_like\({field_pattern},\s*'(?P<{ValueType.regex_value}>(?:[:a-zA-Z\*\?0-9=+%#№;\-_,"\.$&^@!\{{\}}\[\]\s?<>|]|\\\'|\\)+)'\)"""  # noqa: E501
 
     wildcard_symbol = "%"
 
@@ -77,6 +80,22 @@ def create_field_value(field_name: str, operator: Identifier, value: Union[str,
         field_name = field_name.strip('"')
         return FieldValue(source_name=field_name, operator=operator, value=value)
 
+    def _search_re_field_value(self, query: str) -> Optional[tuple[FieldValue, str]]:
+        if match := re.match(self.re_field_value_pattern, query, re.IGNORECASE):
+            group_dict = match.groupdict()
+            field_name = group_dict["field_name"]
+            value = self.str_value_manager.from_re_str_to_container(group_dict[ValueType.regex_value])
+            operator = Identifier(token_type=OperatorType.REGEX)
+            return self.create_field_value(field_name, operator, value), query[match.end() :]
+
     def tokenize(self, query: str) -> list:
         query = re.sub(r"\s*ESCAPE\s*'.'", "", query)  # remove `ESCAPE 'escape_char'` in LIKE expr
         return super().tokenize(query)
+
+    def _get_next_token(
+        self, query: str
+    ) -> tuple[Union[FieldValue, FunctionValue, Keyword, Identifier, list[Union[FieldValue, Identifier]]], str]:
+        query = query.strip("\n").strip(" ").strip("\n")
+        if search_result := self._search_re_field_value(query):
+            return search_result
+        return super()._get_next_token(query)
