Skip to content

Commit 5f2d770

Browse files
dtsprmdtsprm
committed
merge main
2 parents 10030cd + 5a15552 commit 5f2d770

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+309
-170
lines changed

‎uncoder-core/app/routers/translate.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from fastapi import APIRouter, Body
22

33
from app.models.translation import InfoMessage, OneTranslationData, Platform, TranslatorPlatforms
4+
from app.translator.core.context_vars import return_only_first_query_ctx_var
45
from app.translator.cti_translator import CTITranslator
56
from app.translator.translator import Translator
67

@@ -15,7 +16,9 @@ def translate_one(
1516
source_platform_id: str = Body(..., embed=True),
1617
target_platform_id: str = Body(..., embed=True),
1718
text: str = Body(..., embed=True),
19+
return_only_first_query: bool = False,
1820
) -> OneTranslationData:
21+
return_only_first_query_ctx_var.set(return_only_first_query)
1922
status, data = translator.translate_one(text=text, source=source_platform_id, target=target_platform_id)
2023
if status:
2124
return OneTranslationData(status=status, translation=data, target_platform_id=target_platform_id)
@@ -27,8 +30,11 @@ def translate_one(
2730
@st_router.post("/translate/all", tags=["translator"], description="Generate all translations")
2831
@st_router.post("/translate/all/", include_in_schema=False)
2932
def translate_all(
30-
source_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True)
33+
source_platform_id: str = Body(..., embed=True),
34+
text: str = Body(..., embed=True),
35+
return_only_first_query: bool = False,
3136
) -> list[OneTranslationData]:
37+
return_only_first_query_ctx_var.set(return_only_first_query)
3238
result = translator.translate_all(text=text, source=source_platform_id)
3339
translations = []
3440
for platform_result in result:

‎uncoder-core/app/translator/core/custom_types/functions.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,20 @@ class FunctionType(CustomEnum):
99
min = "min"
1010
sum = "sum"
1111

12-
divide = "divide"
12+
values = "values"
1313

1414
earliest = "earliest"
1515
latest = "latest"
1616

17+
divide = "divide"
18+
1719
lower = "lower"
20+
split = "split"
1821
upper = "upper"
1922

20-
compare_boolean = "compare_boolean"
21-
23+
array_length = "array_length"
24+
compare = "compare"
25+
extract_time = "extract_time"
2226
ipv4_is_in_range = "ipv4_is_in_range"
2327

2428
bin = "bin"
@@ -30,4 +34,6 @@ class FunctionType(CustomEnum):
3034
stats = "stats"
3135
table = "table"
3236
timeframe = "timeframe"
33-
values = "values"
37+
union = "union"
38+
39+
aggregation_data_function = "aggregation_data_function"

‎uncoder-core/app/translator/core/custom_types/time.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
from app.translator.tools.custom_enum import CustomEnum
2+
3+
4+
class TimeFrameType(CustomEnum):
5+
years = "years"
6+
months = "months"
7+
days = "days"
8+
hours = "hours"
9+
minutes = "minutes"
10+
11+
12+
class TimePartType(CustomEnum):
13+
day = "day"
14+
day_of_week = "day_of_week"
15+
day_of_year = "day_of_year"
16+
hour = "hour"
17+
microsecond = "microsecond"
18+
millisecond = "millisecond"
19+
minute = "minute"
20+
month = "month"
21+
quarter = "quarter"
22+
second = "second"
23+
year = "year"

‎uncoder-core/app/translator/core/exceptions/core.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
from typing import Optional
2+
3+
14
class NotImplementedException(BaseException):
25
...
36

@@ -7,8 +10,19 @@ class BasePlatformException(BaseException):
710

811

912
class StrictPlatformException(BasePlatformException):
10-
def __init__(self, platform_name: str, field_name: str):
11-
message = f"Platform {platform_name} has strict mapping. Source field {field_name} has no mapping."
13+
field_name: str = None
14+
15+
def __init__(
16+
self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None
17+
):
18+
message = (
19+
f"Platform {platform_name} has strict mapping. "
20+
f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping."
21+
f" Mapping file: {mapping}."
22+
if mapping
23+
else ""
24+
)
25+
self.field_name = field_name
1226
super().__init__(message)
1327

1428

‎uncoder-core/app/translator/core/functions.py

Lines changed: 75 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121
import re
2222
from abc import ABC, abstractmethod
2323
from dataclasses import dataclass
24-
from functools import cached_property
25-
from typing import TYPE_CHECKING, Any, Optional
24+
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
2625

27-
from app.translator.core.exceptions.functions import InvalidFunctionSignature, NotSupportedFunctionException
26+
from app.translator.core.exceptions.functions import NotSupportedFunctionException
2827
from app.translator.core.mapping import SourceMapping
29-
from app.translator.core.models.field import Field
28+
from app.translator.core.models.field import Alias, Field
3029
from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions
30+
from app.translator.tools.utils import execute_module
3131
from settings import INIT_FUNCTIONS
3232

3333
if TYPE_CHECKING:
@@ -41,6 +41,14 @@ class FunctionMatchContainer:
4141

4242

4343
class BaseFunctionParser(ABC):
44+
function_names_map: ClassVar[dict[str, str]] = {}
45+
functions_group_name: str = None
46+
manager: PlatformFunctionsManager = None
47+
48+
def set_functions_manager(self, manager: PlatformFunctionsManager) -> BaseFunctionParser:
49+
self.manager = manager
50+
return self
51+
4452
@abstractmethod
4553
def parse(self, *args, **kwargs) -> Function:
4654
raise NotImplementedError
@@ -73,21 +81,25 @@ def parse(self, func_body: str, raw: str) -> Function:
7381

7482

7583
class FunctionRender(ABC):
84+
function_names_map: ClassVar[dict[str, str]] = {}
85+
order_to_render: int = 0
86+
in_query_render: bool = False
87+
render_to_prefix: bool = False
88+
manager: PlatformFunctionsManager = None
89+
90+
def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRender:
91+
self.manager = manager
92+
return self
93+
7694
@abstractmethod
7795
def render(self, function: Function, source_mapping: SourceMapping) -> str:
7896
raise NotImplementedError
7997

8098
@staticmethod
81-
def concat_kwargs(kwargs: dict[str, str]) -> str:
82-
result = ""
83-
for key, value in kwargs.items():
84-
if value:
85-
result = f"{result}, {key}={value}" if result else f"{key}={value}"
99+
def map_field(field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
100+
if isinstance(field, Alias):
101+
return field.name
86102

87-
return result
88-
89-
@staticmethod
90-
def map_field(field: Field, source_mapping: SourceMapping) -> str:
91103
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
92104
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
93105
if isinstance(mapped_field, list):
@@ -97,23 +109,48 @@ def map_field(field: Field, source_mapping: SourceMapping) -> str:
97109

98110

99111
class PlatformFunctionsManager:
112+
platform_functions: PlatformFunctions = None
113+
100114
def __init__(self):
101-
self._parsers_map: dict[str, HigherOrderFunctionParser] = {}
102-
self._renders_map: dict[str, FunctionRender] = {}
103-
self._in_query_renders_map: dict[str, FunctionRender] = {}
104-
self._names_map: dict[str, str] = {}
105-
self._order_to_render: dict[str, int] = {}
106-
self._render_to_prefix_functions: list[str] = []
107-
108-
def post_init_configure(self, platform_render: PlatformQueryRender) -> None:
109-
raise NotImplementedError
115+
# {platform_func_name: HigherOrderFunctionParser}
116+
self._hof_parsers_map: dict[str, HigherOrderFunctionParser] = {}
117+
self._parsers_map: dict[str, FunctionParser] = {} # {platform_func_name: FunctionParser}
118+
119+
self._renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
120+
self._in_query_renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
121+
self._order_to_render: dict[str, int] = {} # {generic_func_name: int}
122+
123+
def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRender]:
124+
render = render_class()
125+
render.manager = self
126+
for generic_function_name in render.function_names_map:
127+
self._renders_map[generic_function_name] = render
128+
self._order_to_render[generic_function_name] = render.order_to_render
129+
if render.in_query_render:
130+
self._in_query_renders_map[generic_function_name] = render
131+
132+
return render_class
133+
134+
def register_parser(self, parser_class: type[BaseFunctionParser]) -> type[BaseFunctionParser]:
135+
parser = parser_class()
136+
parser.manager = self
137+
parsers_map = self._hof_parsers_map if isinstance(parser, HigherOrderFunctionParser) else self._parsers_map
138+
for platform_function_name in parser.function_names_map:
139+
parsers_map[platform_function_name] = parser
140+
141+
if parser.functions_group_name:
142+
parsers_map[parser.functions_group_name] = parser
143+
144+
return parser_class
145+
146+
def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser:
147+
if INIT_FUNCTIONS and (parser := self._hof_parsers_map.get(platform_func_name)):
148+
return parser
110149

111-
@cached_property
112-
def _inverted_names_map(self) -> dict[str, str]:
113-
return {value: key for key, value in self._names_map.items()}
150+
raise NotSupportedFunctionException
114151

115-
def get_parser(self, generic_func_name: str) -> HigherOrderFunctionParser:
116-
if INIT_FUNCTIONS and (parser := self._parsers_map.get(generic_func_name)):
152+
def get_parser(self, platform_func_name: str) -> FunctionParser:
153+
if INIT_FUNCTIONS and (parser := self._parsers_map.get(platform_func_name)):
117154
return parser
118155

119156
raise NotSupportedFunctionException
@@ -130,56 +167,29 @@ def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
130167

131168
raise NotSupportedFunctionException
132169

133-
def get_generic_func_name(self, platform_func_name: str) -> Optional[str]:
134-
if INIT_FUNCTIONS and (generic_func_name := self._names_map.get(platform_func_name)):
135-
return generic_func_name
136-
137-
raise NotSupportedFunctionException
138-
139-
def get_platform_func_name(self, generic_func_name: str) -> Optional[str]:
140-
if INIT_FUNCTIONS:
141-
return self._inverted_names_map.get(generic_func_name)
142-
143170
@property
144171
def order_to_render(self) -> dict[str, int]:
145172
if INIT_FUNCTIONS:
146173
return self._order_to_render
147174

148175
return {}
149176

150-
@property
151-
def render_to_prefix_functions(self) -> list[str]:
152-
if INIT_FUNCTIONS:
153-
return self._render_to_prefix_functions
154-
155-
return []
156-
157177

158178
class PlatformFunctions:
179+
dir_path: str = None
180+
platform_query_render: PlatformQueryRender = None
159181
manager: PlatformFunctionsManager = PlatformFunctionsManager()
182+
160183
function_delimiter = "|"
161184

162-
def parse(self, query: str) -> ParsedFunctions:
163-
parsed = []
164-
not_supported = []
165-
invalid = []
166-
functions = query.split(self.function_delimiter)
167-
for func in functions:
168-
split_func = func.strip().split(" ")
169-
func_name, func_body = split_func[0], " ".join(split_func[1:])
170-
try:
171-
func_parser = self.manager.get_parser(self.manager.get_generic_func_name(func_name))
172-
parsed.append(func_parser.parse(func_body, func))
173-
except NotSupportedFunctionException:
174-
not_supported.append(func)
175-
except InvalidFunctionSignature:
176-
invalid.append(func)
185+
def __init__(self):
186+
self.manager.platform_functions = self
187+
if self.dir_path:
188+
execute_module(f"{self.dir_path}/parsers/__init__.py")
189+
execute_module(f"{self.dir_path}/renders/__init__.py")
177190

178-
return ParsedFunctions(
179-
functions=parsed,
180-
not_supported=[self.wrap_function_with_delimiter(func) for func in not_supported],
181-
invalid=invalid,
182-
)
191+
def parse(self, query: str) -> ParsedFunctions: # noqa: ARG002
192+
return ParsedFunctions()
183193

184194
def _sort_functions_to_render(self, functions: list[Function]) -> list[Function]:
185195
return sorted(functions, key=lambda func: self.manager.order_to_render.get(func.name, 0))
@@ -193,7 +203,7 @@ def render(self, functions: list[Function], source_mapping: SourceMapping) -> Re
193203
try:
194204
func_render = self.manager.get_render(func.name)
195205
_rendered = func_render.render(func, source_mapping)
196-
if func.name in self.manager.render_to_prefix_functions:
206+
if func_render.render_to_prefix:
197207
rendered_prefix += _rendered
198208
else:
199209
rendered += self.wrap_function_with_delimiter(_rendered)

‎uncoder-core/app/translator/core/parser.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1616
-----------------------------------------------------------------
1717
"""
18+
1819
import re
1920
from abc import ABC, abstractmethod
2021
from typing import Union

‎uncoder-core/app/translator/core/render.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ class PlatformQueryRender(QueryRender):
197197
not_token = "not"
198198

199199
group_token = "(%s)"
200+
query_parts_delimiter = " "
200201

201202
field_value_map = BaseQueryFieldValue(or_token=or_token)
202203

@@ -262,8 +263,14 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
262263

263264
def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str:
264265
result_values = []
266+
unmapped_fields = set()
265267
for token in tokens:
266-
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
268+
try:
269+
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
270+
except StrictPlatformException as err:
271+
unmapped_fields.add(err.field_name)
272+
if unmapped_fields:
273+
raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields))
267274
return "".join(result_values)
268275

269276
def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str:
@@ -284,6 +291,10 @@ def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) ->
284291
def _finalize_search_query(query: str) -> str:
285292
return query
286293

294+
def _join_query_parts(self, prefix: str, query: str, functions: str) -> str:
295+
parts = filter(lambda s: bool(s), map(str.strip, [prefix, self._finalize_search_query(query), functions]))
296+
return self.query_parts_delimiter.join(parts)
297+
287298
def finalize_query(
288299
self,
289300
prefix: str,
@@ -295,8 +306,7 @@ def finalize_query(
295306
*args, # noqa: ARG002
296307
**kwargs, # noqa: ARG002
297308
) -> str:
298-
parts = filter(lambda s: bool(s), map(str.strip, [prefix, self._finalize_search_query(query), functions]))
299-
query = " ".join(parts)
309+
query = self._join_query_parts(prefix, query, functions)
300310
query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query)
301311
if not_supported_functions:
302312
rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
@@ -383,7 +393,7 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
383393
defined_raw_log_fields = self.generate_raw_log_fields(
384394
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
385395
)
386-
prefix += f"\n{defined_raw_log_fields}\n"
396+
prefix += f"\n{defined_raw_log_fields}"
387397
result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
388398
except StrictPlatformException as err:
389399
errors.append(err)

‎uncoder-core/app/translator/core/render_cti.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
-----------------------------------------------------------------
1818
"""
1919

20-
2120
from app.translator.core.models.iocs import IocsChunkValue
2221
from app.translator.core.models.platform_details import PlatformDetails
2322

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,3 +125,4 @@ field_mapping:
125125
SourceOS: xdm.source.host.os
126126
DestinationOS: xdm.target.host.os
127127
url_category: xdm.network.http.url_category
128+
EventSeverity: xdm.alert.severity

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy