resolve conflicts

oleksandr.volha · oleksandr.volha · commit 4b9e590cd6f0 · 2024-05-09T17:17:09.000+03:00
diff --git a/uncoder-core/app/translator/core/custom_types/functions.py b/uncoder-core/app/translator/core/custom_types/functions.py
@@ -4,10 +4,17 @@
 class FunctionType(CustomEnum):
     avg = "avg"
     count = "count"
+    distinct_count = "distinct_count"
+    earliest = "earliest"
+    eval = "eval"
+    fields = "fields"
+    latest = "latest"
     max = "max"
     min = "min"
+    rename = "rename"
     search = "search"
     sort = "sort"
     stats = "stats"
     sum = "sum"
     table = "table"
+    values = "values"
diff --git a/uncoder-core/app/translator/core/custom_types/tokens.py b/uncoder-core/app/translator/core/custom_types/tokens.py
@@ -23,6 +23,8 @@ class OperatorType(CustomEnum):
     REGEX = "re"
     NOT_REGEX = "not re"
     KEYWORD = "keyword"
+    IS_NONE = "is none"
+    IS_NOT_NONE = "is not none"
 
 
 class GroupType(CustomEnum):
diff --git a/uncoder-core/app/translator/core/functions.py b/uncoder-core/app/translator/core/functions.py
@@ -18,24 +18,55 @@
 
 from __future__ import annotations
 
+import re
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from functools import cached_property
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Any, Optional
 
 from app.translator.core.exceptions.functions import InvalidFunctionSignature, NotSupportedFunctionException
 from app.translator.core.mapping import SourceMapping
 from app.translator.core.models.field import Field
 from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions
-from app.translator.core.tokenizer import BaseTokenizer
 from settings import INIT_FUNCTIONS
 
 if TYPE_CHECKING:
     from app.translator.core.render import PlatformQueryRender
 
 
-class FunctionParser(ABC):
-    tokenizer: BaseTokenizer = None
+@dataclass
+class FunctionMatchContainer:
+    name: str
+    match: re.Match
 
+
+class BaseFunctionParser(ABC):
+    @abstractmethod
+    def parse(self, *args, **kwargs) -> Function:
+        raise NotImplementedError
+
+    def tokenize_body(self, func_body: str) -> list[Any]:
+        tokenized = []
+        while func_body:
+            identifier, func_body = self._get_next_token(func_body=func_body)
+            tokenized.append(identifier)
+        return tokenized
+
+    def _get_next_token(self, func_body: str) -> tuple[Any, str]:
+        raise NotImplementedError
+
+
+class FunctionParser(BaseFunctionParser):
+    @abstractmethod
+    def parse(self, func_name: str, match: re.Match) -> Function:
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_func_match(self, higher_order_func_body: str) -> Optional[FunctionMatchContainer]:
+        raise NotImplementedError
+
+
+class HigherOrderFunctionParser(BaseFunctionParser):  # for highest level functions e.g. | stats, | search, | sort, etc.
     @abstractmethod
     def parse(self, func_body: str, raw: str) -> Function:
         raise NotImplementedError
@@ -67,18 +98,18 @@ def map_field(field: Field, source_mapping: SourceMapping) -> str:
 
 class PlatformFunctionsManager:
     def __init__(self):
-        self._parsers_map: dict[str, FunctionParser] = {}
+        self._parsers_map: dict[str, HigherOrderFunctionParser] = {}
         self._renders_map: dict[str, FunctionRender] = {}
         self._names_map: dict[str, str] = {}
 
-    def init_search_func_render(self, platform_render: PlatformQueryRender) -> None:
+    def post_init_configure(self, platform_render: PlatformQueryRender) -> None:
         raise NotImplementedError
 
     @cached_property
     def _inverted_names_map(self) -> dict[str, str]:
         return {value: key for key, value in self._names_map.items()}
 
-    def get_parser(self, generic_func_name: str) -> FunctionParser:
+    def get_parser(self, generic_func_name: str) -> HigherOrderFunctionParser:
         if INIT_FUNCTIONS and (parser := self._parsers_map.get(generic_func_name)):
             return parser
 
diff --git a/uncoder-core/app/translator/core/models/functions/eval.py b/uncoder-core/app/translator/core/models/functions/eval.py
@@ -0,0 +1,19 @@
+from dataclasses import dataclass, field
+from typing import Union
+
+from app.translator.core.custom_types.functions import FunctionType
+from app.translator.core.models.field import Field
+from app.translator.core.models.functions.base import Function
+from app.translator.core.models.identifier import Identifier
+
+
+@dataclass
+class EvalArg:
+    field_: Field = None
+    expression: list[Union[Field, Function, Identifier, int, float, str]] = field(default_factory=list)
+
+
+@dataclass
+class EvalFunction(Function):
+    name: str = FunctionType.eval
+    args: list[EvalArg] = None
diff --git a/uncoder-core/app/translator/core/models/functions/rename.py b/uncoder-core/app/translator/core/models/functions/rename.py
@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+
+from app.translator.core.custom_types.functions import FunctionType
+from app.translator.core.models.field import Field
+from app.translator.core.models.functions.base import Function
+
+
+@dataclass
+class RenameArg:
+    field_: Field = None
+    alias: str = None
+
+
+@dataclass
+class RenameFunction(Function):
+    name: str = FunctionType.rename
+    args: list[RenameArg] = None
diff --git a/uncoder-core/app/translator/platforms/base/spl/functions/const.py b/uncoder-core/app/translator/platforms/base/spl/functions/const.py
@@ -4,13 +4,21 @@
 class SplFunctionType(CustomEnum):
     avg = "avg"
     count = "count"
+    distinct_count = "distinct_count"
+    earliest = "earliest"
+    eval = "eval"
+    fields = "fields"
+    latest = "latest"
     max = "max"
     min = "min"
+    rename = "rename"
     search = "search"
     sort = "sort"
     stats = "stats"
     sum = "sum"
     table = "table"
+    values = "values"
+    where = "where"
 
 
 class SplSortOrderType(CustomEnum):
diff --git a/uncoder-core/app/translator/platforms/base/spl/functions/manager.py b/uncoder-core/app/translator/platforms/base/spl/functions/manager.py
@@ -9,5 +9,5 @@
 
 
 class SplFunctionsManager(PlatformFunctionsManager):
-    def init_search_func_render(self, platform_render: SplQueryRender) -> None:
+    def post_init_configure(self, platform_render: SplQueryRender) -> None:
         pass
diff --git a/uncoder-core/app/translator/platforms/logscale/functions/manager.py b/uncoder-core/app/translator/platforms/logscale/functions/manager.py
@@ -9,5 +9,5 @@
 
 
 class LogScaleFunctionsManager(PlatformFunctionsManager):
-    def init_search_func_render(self, platform_render: LogScaleQueryRender) -> None:
+    def post_init_configure(self, platform_render: LogScaleQueryRender) -> None:
         pass
diff --git a/uncoder-core/app/translator/platforms/microsoft/functions/const.py b/uncoder-core/app/translator/platforms/microsoft/functions/const.py
@@ -4,11 +4,21 @@
 class KQLFunctionType(CustomEnum):
     avg = "avg"
     count = "count"
+    distinct = "distinct"
+    distinct_count = "count_distinct"
+    extend = "extend"
     max = "max"
     min = "min"
-    sum = "sum"
-    where = "where"
-    search = "search"
-    summarize = "summarize"
     project = "project"
+    project_rename = "project-rename"
+    search = "search"
     sort = "sort"
+    sum = "sum"
+    summarize = "summarize"
+    top = "top"
+    where = "where"
+
+
+class KQLSortOrderType(CustomEnum):
+    asc = "asc"
+    desc = "desc"
diff --git a/uncoder-core/app/translator/platforms/microsoft/functions/manager.py b/uncoder-core/app/translator/platforms/microsoft/functions/manager.py
@@ -9,5 +9,5 @@
 
 
 class MicrosoftFunctionsManager(PlatformFunctionsManager):
-    def init_search_func_render(self, platform_render: MicrosoftSentinelQueryRender) -> None:
+    def post_init_configure(self, platform_render: MicrosoftSentinelQueryRender) -> None:
         pass
