Skip to content

Commit 8a06b99

Browse files
alexvolhaalexvolha
authored
Merge pull request #152 from UncoderIO/gis-7984
Gis 7984
2 parents cf48a62 + 13298c9 commit 8a06b99

File tree

72 files changed

+353
-238
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+353
-238
lines changed

‎uncoder-core/app/translator/core/context_vars.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
from contextvars import ContextVar
22

33
return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False)
4-
"""Set to True to return ony first query if rendered multiple options"""
4+
"""Set to True to return only first query if rendered multiple options"""

‎uncoder-core/app/translator/core/custom_types/functions.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,20 @@ class FunctionType(CustomEnum):
99
min = "min"
1010
sum = "sum"
1111

12-
divide = "divide"
12+
values = "values"
1313

1414
earliest = "earliest"
1515
latest = "latest"
1616

17+
divide = "divide"
18+
1719
lower = "lower"
20+
split = "split"
1821
upper = "upper"
1922

20-
compare_boolean = "compare_boolean"
21-
23+
array_length = "array_length"
24+
compare = "compare"
25+
extract_time = "extract_time"
2226
ipv4_is_in_range = "ipv4_is_in_range"
2327

2428
bin = "bin"
@@ -30,4 +34,6 @@ class FunctionType(CustomEnum):
3034
stats = "stats"
3135
table = "table"
3236
timeframe = "timeframe"
33-
values = "values"
37+
union = "union"
38+
39+
aggregation_data_function = "aggregation_data_function"

‎uncoder-core/app/translator/core/custom_types/time.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
from app.translator.tools.custom_enum import CustomEnum
2+
3+
4+
class TimeFrameType(CustomEnum):
5+
years = "years"
6+
months = "months"
7+
days = "days"
8+
hours = "hours"
9+
minutes = "minutes"

‎uncoder-core/app/translator/core/functions.py

Lines changed: 75 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121
import re
2222
from abc import ABC, abstractmethod
2323
from dataclasses import dataclass
24-
from functools import cached_property
25-
from typing import TYPE_CHECKING, Any, Optional
24+
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
2625

27-
from app.translator.core.exceptions.functions import InvalidFunctionSignature, NotSupportedFunctionException
26+
from app.translator.core.exceptions.functions import NotSupportedFunctionException
2827
from app.translator.core.mapping import SourceMapping
29-
from app.translator.core.models.field import Field
28+
from app.translator.core.models.field import Alias, Field
3029
from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions
30+
from app.translator.tools.utils import execute_module
3131
from settings import INIT_FUNCTIONS
3232

3333
if TYPE_CHECKING:
@@ -41,6 +41,14 @@ class FunctionMatchContainer:
4141

4242

4343
class BaseFunctionParser(ABC):
44+
function_names_map: ClassVar[dict[str, str]] = {}
45+
functions_group_name: str = None
46+
manager: PlatformFunctionsManager = None
47+
48+
def set_functions_manager(self, manager: PlatformFunctionsManager) -> BaseFunctionParser:
49+
self.manager = manager
50+
return self
51+
4452
@abstractmethod
4553
def parse(self, *args, **kwargs) -> Function:
4654
raise NotImplementedError
@@ -73,21 +81,25 @@ def parse(self, func_body: str, raw: str) -> Function:
7381

7482

7583
class FunctionRender(ABC):
84+
function_names_map: ClassVar[dict[str, str]] = {}
85+
order_to_render: int = 0
86+
in_query_render: bool = False
87+
render_to_prefix: bool = False
88+
manager: PlatformFunctionsManager = None
89+
90+
def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRender:
91+
self.manager = manager
92+
return self
93+
7694
@abstractmethod
7795
def render(self, function: Function, source_mapping: SourceMapping) -> str:
7896
raise NotImplementedError
7997

8098
@staticmethod
81-
def concat_kwargs(kwargs: dict[str, str]) -> str:
82-
result = ""
83-
for key, value in kwargs.items():
84-
if value:
85-
result = f"{result}, {key}={value}" if result else f"{key}={value}"
99+
def map_field(field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
100+
if isinstance(field, Alias):
101+
return field.name
86102

87-
return result
88-
89-
@staticmethod
90-
def map_field(field: Field, source_mapping: SourceMapping) -> str:
91103
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
92104
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
93105
if isinstance(mapped_field, list):
@@ -97,23 +109,48 @@ def map_field(field: Field, source_mapping: SourceMapping) -> str:
97109

98110

99111
class PlatformFunctionsManager:
112+
platform_functions: PlatformFunctions = None
113+
100114
def __init__(self):
101-
self._parsers_map: dict[str, HigherOrderFunctionParser] = {}
102-
self._renders_map: dict[str, FunctionRender] = {}
103-
self._in_query_renders_map: dict[str, FunctionRender] = {}
104-
self._names_map: dict[str, str] = {}
105-
self._order_to_render: dict[str, int] = {}
106-
self._render_to_prefix_functions: list[str] = []
107-
108-
def post_init_configure(self, platform_render: PlatformQueryRender) -> None:
109-
raise NotImplementedError
115+
# {platform_func_name: HigherOrderFunctionParser}
116+
self._hof_parsers_map: dict[str, HigherOrderFunctionParser] = {}
117+
self._parsers_map: dict[str, FunctionParser] = {} # {platform_func_name: FunctionParser}
118+
119+
self._renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
120+
self._in_query_renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
121+
self._order_to_render: dict[str, int] = {} # {generic_func_name: int}
122+
123+
def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRender]:
124+
render = render_class()
125+
render.manager = self
126+
for generic_function_name in render.function_names_map:
127+
self._renders_map[generic_function_name] = render
128+
self._order_to_render[generic_function_name] = render.order_to_render
129+
if render.in_query_render:
130+
self._in_query_renders_map[generic_function_name] = render
131+
132+
return render_class
133+
134+
def register_parser(self, parser_class: type[BaseFunctionParser]) -> type[BaseFunctionParser]:
135+
parser = parser_class()
136+
parser.manager = self
137+
parsers_map = self._hof_parsers_map if isinstance(parser, HigherOrderFunctionParser) else self._parsers_map
138+
for platform_function_name in parser.function_names_map:
139+
parsers_map[platform_function_name] = parser
140+
141+
if parser.functions_group_name:
142+
parsers_map[parser.functions_group_name] = parser
143+
144+
return parser_class
145+
146+
def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser:
147+
if INIT_FUNCTIONS and (parser := self._hof_parsers_map.get(platform_func_name)):
148+
return parser
110149

111-
@cached_property
112-
def _inverted_names_map(self) -> dict[str, str]:
113-
return {value: key for key, value in self._names_map.items()}
150+
raise NotSupportedFunctionException
114151

115-
def get_parser(self, generic_func_name: str) -> HigherOrderFunctionParser:
116-
if INIT_FUNCTIONS and (parser := self._parsers_map.get(generic_func_name)):
152+
def get_parser(self, platform_func_name: str) -> FunctionParser:
153+
if INIT_FUNCTIONS and (parser := self._parsers_map.get(platform_func_name)):
117154
return parser
118155

119156
raise NotSupportedFunctionException
@@ -130,56 +167,29 @@ def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
130167

131168
raise NotSupportedFunctionException
132169

133-
def get_generic_func_name(self, platform_func_name: str) -> Optional[str]:
134-
if INIT_FUNCTIONS and (generic_func_name := self._names_map.get(platform_func_name)):
135-
return generic_func_name
136-
137-
raise NotSupportedFunctionException
138-
139-
def get_platform_func_name(self, generic_func_name: str) -> Optional[str]:
140-
if INIT_FUNCTIONS:
141-
return self._inverted_names_map.get(generic_func_name)
142-
143170
@property
144171
def order_to_render(self) -> dict[str, int]:
145172
if INIT_FUNCTIONS:
146173
return self._order_to_render
147174

148175
return {}
149176

150-
@property
151-
def render_to_prefix_functions(self) -> list[str]:
152-
if INIT_FUNCTIONS:
153-
return self._render_to_prefix_functions
154-
155-
return []
156-
157177

158178
class PlatformFunctions:
179+
dir_path: str = None
180+
platform_query_render: PlatformQueryRender = None
159181
manager: PlatformFunctionsManager = PlatformFunctionsManager()
182+
160183
function_delimiter = "|"
161184

162-
def parse(self, query: str) -> ParsedFunctions:
163-
parsed = []
164-
not_supported = []
165-
invalid = []
166-
functions = query.split(self.function_delimiter)
167-
for func in functions:
168-
split_func = func.strip().split(" ")
169-
func_name, func_body = split_func[0], " ".join(split_func[1:])
170-
try:
171-
func_parser = self.manager.get_parser(self.manager.get_generic_func_name(func_name))
172-
parsed.append(func_parser.parse(func_body, func))
173-
except NotSupportedFunctionException:
174-
not_supported.append(func)
175-
except InvalidFunctionSignature:
176-
invalid.append(func)
185+
def __init__(self):
186+
self.manager.platform_functions = self
187+
if self.dir_path:
188+
execute_module(f"{self.dir_path}/parsers/__init__.py")
189+
execute_module(f"{self.dir_path}/renders/__init__.py")
177190

178-
return ParsedFunctions(
179-
functions=parsed,
180-
not_supported=[self.wrap_function_with_delimiter(func) for func in not_supported],
181-
invalid=invalid,
182-
)
191+
def parse(self, query: str) -> ParsedFunctions: # noqa: ARG002
192+
return ParsedFunctions()
183193

184194
def _sort_functions_to_render(self, functions: list[Function]) -> list[Function]:
185195
return sorted(functions, key=lambda func: self.manager.order_to_render.get(func.name, 0))
@@ -193,7 +203,7 @@ def render(self, functions: list[Function], source_mapping: SourceMapping) -> Re
193203
try:
194204
func_render = self.manager.get_render(func.name)
195205
_rendered = func_render.render(func, source_mapping)
196-
if func.name in self.manager.render_to_prefix_functions:
206+
if func_render.render_to_prefix:
197207
rendered_prefix += _rendered
198208
else:
199209
rendered += self.wrap_function_with_delimiter(_rendered)

‎uncoder-core/app/translator/core/parser.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1616
-----------------------------------------------------------------
1717
"""
18+
1819
import re
1920
from abc import ABC, abstractmethod
2021
from typing import Union

‎uncoder-core/app/translator/core/render.py

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
1920
from abc import ABC, abstractmethod
2021
from collections.abc import Callable
21-
from typing import Optional, Union
22+
from typing import ClassVar, Optional, Union
2223

2324
from app.translator.const import DEFAULT_VALUE_TYPE
2425
from app.translator.core.context_vars import return_only_first_query_ctx_var
@@ -165,7 +166,14 @@ class QueryRender(ABC):
165166
is_single_line_comment: bool = False
166167
unsupported_functions_text = "Unsupported functions were excluded from the result query:"
167168

168-
platform_functions: PlatformFunctions = PlatformFunctions()
169+
platform_functions: PlatformFunctions = None
170+
171+
def __init__(self):
172+
self.init_platform_functions()
173+
174+
def init_platform_functions(self) -> None:
175+
self.platform_functions = PlatformFunctions()
176+
self.platform_functions.platform_query_render = self
169177

170178
def render_not_supported_functions(self, not_supported_functions: list) -> str:
171179
line_template = f"{self.comment_symbol} " if self.comment_symbol and self.is_single_line_comment else ""
@@ -192,19 +200,19 @@ class PlatformQueryRender(QueryRender):
192200

193201
field_value_map = BaseQueryFieldValue(or_token=or_token)
194202

195-
query_pattern = "{table} {query} {functions}"
196-
raw_log_field_pattern_map: dict = None
203+
raw_log_field_pattern_map: ClassVar[dict[str, str]] = None
197204

198205
def __init__(self):
206+
super().__init__()
199207
self.operator_map = {
200208
LogicalOperatorType.AND: f" {self.and_token} ",
201209
LogicalOperatorType.OR: f" {self.or_token} ",
202210
LogicalOperatorType.NOT: f" {self.not_token} ",
203211
}
204212

205-
def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002
206-
if str(log_source_signature):
207-
return f"{log_source_signature!s} {self.and_token}"
213+
def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002
214+
if log_source_signature and str(log_source_signature):
215+
return f"{log_source_signature} {self.and_token}"
208216
return ""
209217

210218
def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions:
@@ -272,6 +280,10 @@ def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) ->
272280
query = f"{query}\n\n{query_meta_info}"
273281
return query
274282

283+
@staticmethod
284+
def _finalize_search_query(query: str) -> str:
285+
return query
286+
275287
def finalize_query(
276288
self,
277289
prefix: str,
@@ -283,8 +295,8 @@ def finalize_query(
283295
*args, # noqa: ARG002
284296
**kwargs, # noqa: ARG002
285297
) -> str:
286-
query = self.query_pattern.format(prefix=prefix, query=query, functions=functions).strip()
287-
298+
parts = filter(lambda s: bool(s), map(str.strip, [prefix, self._finalize_search_query(query), functions]))
299+
query = " ".join(parts)
288300
query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query)
289301
if not_supported_functions:
290302
rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
@@ -327,15 +339,15 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer)
327339

328340
def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
329341
if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type):
330-
return raw_log_field_pattern.pattern.format(field=field)
342+
return raw_log_field_pattern.format(field=field)
331343

332344
def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[list]:
333345
if isinstance(field, list):
334-
list_of_prefix = []
346+
prefix_list = []
335347
for f in field:
336-
if prepared_prefix := self.process_raw_log_field_prefix(field=f, source_mapping=source_mapping):
337-
list_of_prefix.extend(prepared_prefix)
338-
return list_of_prefix
348+
if _prefix_list := self.process_raw_log_field_prefix(field=f, source_mapping=source_mapping):
349+
prefix_list.extend(_prefix_list)
350+
return prefix_list
339351
if raw_log_field_type := source_mapping.raw_log_fields.get(field):
340352
return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)]
341353

@@ -352,9 +364,11 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
352364
)
353365
if not mapped_field and self.is_strict_mapping:
354366
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
355-
if field_prefix := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
356-
defined_raw_log_fields.extend(field_prefix)
357-
return "\n".join(set(defined_raw_log_fields))
367+
if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
368+
for prefix in prefix_list:
369+
if prefix not in defined_raw_log_fields:
370+
defined_raw_log_fields.append(prefix)
371+
return "\n".join(defined_raw_log_fields)
358372

359373
def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
360374
queries_map = {}

‎uncoder-core/app/translator/core/render_cti.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
-----------------------------------------------------------------
1818
"""
1919

20-
2120
from app.translator.core.models.iocs import IocsChunkValue
2221
from app.translator.core.models.platform_details import PlatformDetails
2322

‎uncoder-core/app/translator/core/str_value_manager.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
1920
from typing import ClassVar, Optional, TypeVar, Union
2021

2122
from app.translator.core.custom_types.values import ValueType

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy