Skip to content

Commit 6f1173b

Browse files
committed
merge main
2 parents 9a9f093 + 8b77550 commit 6f1173b

File tree

26 files changed

+221
-151
lines changed

26 files changed

+221
-151
lines changed

uncoder-core/app/routers/translate.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from fastapi import APIRouter, Body
22

33
from app.models.translation import InfoMessage, OneTranslationData, Platform, TranslatorPlatforms
4+
from app.translator.core.context_vars import return_only_first_query_ctx_var
45
from app.translator.cti_translator import CTITranslator
56
from app.translator.translator import Translator
67

@@ -15,7 +16,9 @@ def translate_one(
1516
source_platform_id: str = Body(..., embed=True),
1617
target_platform_id: str = Body(..., embed=True),
1718
text: str = Body(..., embed=True),
19+
return_only_first_query: bool = False,
1820
) -> OneTranslationData:
21+
return_only_first_query_ctx_var.set(return_only_first_query)
1922
status, data = translator.translate_one(text=text, source=source_platform_id, target=target_platform_id)
2023
if status:
2124
return OneTranslationData(status=status, translation=data, target_platform_id=target_platform_id)
@@ -27,8 +30,11 @@ def translate_one(
2730
@st_router.post("/translate/all", tags=["translator"], description="Generate all translations")
2831
@st_router.post("/translate/all/", include_in_schema=False)
2932
def translate_all(
30-
source_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True)
33+
source_platform_id: str = Body(..., embed=True),
34+
text: str = Body(..., embed=True),
35+
return_only_first_query: bool = False,
3136
) -> list[OneTranslationData]:
37+
return_only_first_query_ctx_var.set(return_only_first_query)
3238
result = translator.translate_all(text=text, source=source_platform_id)
3339
translations = []
3440
for platform_result in result:

uncoder-core/app/translator/core/custom_types/functions.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,20 @@ class FunctionType(CustomEnum):
99
min = "min"
1010
sum = "sum"
1111

12-
divide = "divide"
12+
values = "values"
1313

1414
earliest = "earliest"
1515
latest = "latest"
1616

17+
divide = "divide"
18+
1719
lower = "lower"
20+
split = "split"
1821
upper = "upper"
1922

20-
compare_boolean = "compare_boolean"
21-
23+
array_length = "array_length"
24+
compare = "compare"
25+
extract_time = "extract_time"
2226
ipv4_is_in_range = "ipv4_is_in_range"
2327

2428
bin = "bin"
@@ -30,4 +34,6 @@ class FunctionType(CustomEnum):
3034
stats = "stats"
3135
table = "table"
3236
timeframe = "timeframe"
33-
values = "values"
37+
union = "union"
38+
39+
aggregation_data_function = "aggregation_data_function"

uncoder-core/app/translator/core/functions.py

Lines changed: 75 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121
import re
2222
from abc import ABC, abstractmethod
2323
from dataclasses import dataclass
24-
from functools import cached_property
25-
from typing import TYPE_CHECKING, Any, Optional
24+
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
2625

27-
from app.translator.core.exceptions.functions import InvalidFunctionSignature, NotSupportedFunctionException
26+
from app.translator.core.exceptions.functions import NotSupportedFunctionException
2827
from app.translator.core.mapping import SourceMapping
29-
from app.translator.core.models.field import Field
28+
from app.translator.core.models.field import Alias, Field
3029
from app.translator.core.models.functions.base import Function, ParsedFunctions, RenderedFunctions
30+
from app.translator.tools.utils import execute_module
3131
from settings import INIT_FUNCTIONS
3232

3333
if TYPE_CHECKING:
@@ -41,6 +41,14 @@ class FunctionMatchContainer:
4141

4242

4343
class BaseFunctionParser(ABC):
44+
function_names_map: ClassVar[dict[str, str]] = {}
45+
functions_group_name: str = None
46+
manager: PlatformFunctionsManager = None
47+
48+
def set_functions_manager(self, manager: PlatformFunctionsManager) -> BaseFunctionParser:
49+
self.manager = manager
50+
return self
51+
4452
@abstractmethod
4553
def parse(self, *args, **kwargs) -> Function:
4654
raise NotImplementedError
@@ -73,21 +81,25 @@ def parse(self, func_body: str, raw: str) -> Function:
7381

7482

7583
class FunctionRender(ABC):
84+
function_names_map: ClassVar[dict[str, str]] = {}
85+
order_to_render: int = 0
86+
in_query_render: bool = False
87+
render_to_prefix: bool = False
88+
manager: PlatformFunctionsManager = None
89+
90+
def set_functions_manager(self, manager: PlatformFunctionsManager) -> FunctionRender:
91+
self.manager = manager
92+
return self
93+
7694
@abstractmethod
7795
def render(self, function: Function, source_mapping: SourceMapping) -> str:
7896
raise NotImplementedError
7997

8098
@staticmethod
81-
def concat_kwargs(kwargs: dict[str, str]) -> str:
82-
result = ""
83-
for key, value in kwargs.items():
84-
if value:
85-
result = f"{result}, {key}={value}" if result else f"{key}={value}"
99+
def map_field(field: Union[Alias, Field], source_mapping: SourceMapping) -> str:
100+
if isinstance(field, Alias):
101+
return field.name
86102

87-
return result
88-
89-
@staticmethod
90-
def map_field(field: Field, source_mapping: SourceMapping) -> str:
91103
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
92104
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
93105
if isinstance(mapped_field, list):
@@ -97,23 +109,48 @@ def map_field(field: Field, source_mapping: SourceMapping) -> str:
97109

98110

99111
class PlatformFunctionsManager:
112+
platform_functions: PlatformFunctions = None
113+
100114
def __init__(self):
101-
self._parsers_map: dict[str, HigherOrderFunctionParser] = {}
102-
self._renders_map: dict[str, FunctionRender] = {}
103-
self._in_query_renders_map: dict[str, FunctionRender] = {}
104-
self._names_map: dict[str, str] = {}
105-
self._order_to_render: dict[str, int] = {}
106-
self._render_to_prefix_functions: list[str] = []
107-
108-
def post_init_configure(self, platform_render: PlatformQueryRender) -> None:
109-
raise NotImplementedError
115+
# {platform_func_name: HigherOrderFunctionParser}
116+
self._hof_parsers_map: dict[str, HigherOrderFunctionParser] = {}
117+
self._parsers_map: dict[str, FunctionParser] = {} # {platform_func_name: FunctionParser}
118+
119+
self._renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
120+
self._in_query_renders_map: dict[str, FunctionRender] = {} # {generic_func_name: FunctionRender}
121+
self._order_to_render: dict[str, int] = {} # {generic_func_name: int}
122+
123+
def register_render(self, render_class: type[FunctionRender]) -> type[FunctionRender]:
124+
render = render_class()
125+
render.manager = self
126+
for generic_function_name in render.function_names_map:
127+
self._renders_map[generic_function_name] = render
128+
self._order_to_render[generic_function_name] = render.order_to_render
129+
if render.in_query_render:
130+
self._in_query_renders_map[generic_function_name] = render
131+
132+
return render_class
133+
134+
def register_parser(self, parser_class: type[BaseFunctionParser]) -> type[BaseFunctionParser]:
135+
parser = parser_class()
136+
parser.manager = self
137+
parsers_map = self._hof_parsers_map if isinstance(parser, HigherOrderFunctionParser) else self._parsers_map
138+
for platform_function_name in parser.function_names_map:
139+
parsers_map[platform_function_name] = parser
140+
141+
if parser.functions_group_name:
142+
parsers_map[parser.functions_group_name] = parser
143+
144+
return parser_class
145+
146+
def get_hof_parser(self, platform_func_name: str) -> HigherOrderFunctionParser:
147+
if INIT_FUNCTIONS and (parser := self._hof_parsers_map.get(platform_func_name)):
148+
return parser
110149

111-
@cached_property
112-
def _inverted_names_map(self) -> dict[str, str]:
113-
return {value: key for key, value in self._names_map.items()}
150+
raise NotSupportedFunctionException
114151

115-
def get_parser(self, generic_func_name: str) -> HigherOrderFunctionParser:
116-
if INIT_FUNCTIONS and (parser := self._parsers_map.get(generic_func_name)):
152+
def get_parser(self, platform_func_name: str) -> FunctionParser:
153+
if INIT_FUNCTIONS and (parser := self._parsers_map.get(platform_func_name)):
117154
return parser
118155

119156
raise NotSupportedFunctionException
@@ -130,56 +167,29 @@ def get_in_query_render(self, generic_func_name: str) -> FunctionRender:
130167

131168
raise NotSupportedFunctionException
132169

133-
def get_generic_func_name(self, platform_func_name: str) -> Optional[str]:
134-
if INIT_FUNCTIONS and (generic_func_name := self._names_map.get(platform_func_name)):
135-
return generic_func_name
136-
137-
raise NotSupportedFunctionException
138-
139-
def get_platform_func_name(self, generic_func_name: str) -> Optional[str]:
140-
if INIT_FUNCTIONS:
141-
return self._inverted_names_map.get(generic_func_name)
142-
143170
@property
144171
def order_to_render(self) -> dict[str, int]:
145172
if INIT_FUNCTIONS:
146173
return self._order_to_render
147174

148175
return {}
149176

150-
@property
151-
def render_to_prefix_functions(self) -> list[str]:
152-
if INIT_FUNCTIONS:
153-
return self._render_to_prefix_functions
154-
155-
return []
156-
157177

158178
class PlatformFunctions:
179+
dir_path: str = None
180+
platform_query_render: PlatformQueryRender = None
159181
manager: PlatformFunctionsManager = PlatformFunctionsManager()
182+
160183
function_delimiter = "|"
161184

162-
def parse(self, query: str) -> ParsedFunctions:
163-
parsed = []
164-
not_supported = []
165-
invalid = []
166-
functions = query.split(self.function_delimiter)
167-
for func in functions:
168-
split_func = func.strip().split(" ")
169-
func_name, func_body = split_func[0], " ".join(split_func[1:])
170-
try:
171-
func_parser = self.manager.get_parser(self.manager.get_generic_func_name(func_name))
172-
parsed.append(func_parser.parse(func_body, func))
173-
except NotSupportedFunctionException:
174-
not_supported.append(func)
175-
except InvalidFunctionSignature:
176-
invalid.append(func)
185+
def __init__(self):
186+
self.manager.platform_functions = self
187+
if self.dir_path:
188+
execute_module(f"{self.dir_path}/parsers/__init__.py")
189+
execute_module(f"{self.dir_path}/renders/__init__.py")
177190

178-
return ParsedFunctions(
179-
functions=parsed,
180-
not_supported=[self.wrap_function_with_delimiter(func) for func in not_supported],
181-
invalid=invalid,
182-
)
191+
def parse(self, query: str) -> ParsedFunctions: # noqa: ARG002
192+
return ParsedFunctions()
183193

184194
def _sort_functions_to_render(self, functions: list[Function]) -> list[Function]:
185195
return sorted(functions, key=lambda func: self.manager.order_to_render.get(func.name, 0))
@@ -193,7 +203,7 @@ def render(self, functions: list[Function], source_mapping: SourceMapping) -> Re
193203
try:
194204
func_render = self.manager.get_render(func.name)
195205
_rendered = func_render.render(func, source_mapping)
196-
if func.name in self.manager.render_to_prefix_functions:
206+
if func_render.render_to_prefix:
197207
rendered_prefix += _rendered
198208
else:
199209
rendered += self.wrap_function_with_delimiter(_rendered)

uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@ field_mapping:
1010
#dns-record: dns-record
1111
dns_query_name: xdm.network.dns.dns_question.name
1212
QueryName: xdm.network.dns.dns_question.name
13-
query: xdm.network.dns.dns_question.name
13+
query: xdm.network.dns.dns_question.name
14+
dns-record-type: xdm.network.dns.dns_question.type

uncoder-core/app/translator/mappings/platforms/qradar/default.yml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@ field_mapping:
3535
- userName
3636
- EventUserName
3737
CommandLine: Command
38-
Protocol: IPProtocol
38+
Protocol:
39+
- IPProtocol
40+
- protocol
3941
Application:
4042
- Application
4143
- application
@@ -61,6 +63,7 @@ field_mapping:
6163
SourceMAC:
6264
- SourceMAC
6365
- MAC
66+
- sourceMAC
6467
DestinationMAC: DestinationMAC
6568
SourceOS:
6669
- SourceOS
@@ -69,4 +72,7 @@ field_mapping:
6972
TargetUserName: DestinationUserName
7073
SourceUserName: SourceUserName
7174
url_category: XForceCategoryByURL
72-
EventSeverity: EventSeverity
75+
EventSeverity: EventSeverity
76+
Source:
77+
- Source
78+
- source

uncoder-core/app/translator/mappings/platforms/qradar/dns.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@ field_mapping:
1212
dns-query: URL
1313
parent-domain: parent-domain
1414
dns-answer: dns-answer
15-
dns-record: URL
15+
dns-record: URL
16+
dns-record-type: DNSRecordType

uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ field_mapping:
2424
cs-host:
2525
- UrlHost
2626
- URL Host
27+
- URL Domain
2728
cs-referrer:
2829
- URL Referrer
2930
- Referrer URL

uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ field_mapping:
4141
LinkName: LinkName
4242
MemberName: MemberName
4343
MemberSid: MemberSid
44-
NewProcessName: Process Name
44+
NewProcessName:
45+
- Process Name
46+
- New Process Name
4547
ObjectClass: ObjectClass
4648
ObjectName:
4749
- Object Name
@@ -122,6 +124,7 @@ field_mapping:
122124
ServiceFileName:
123125
- Service Filename
124126
- ServiceFileName
127+
- Service File Name
125128
SecurityDescriptor: SecurityDescriptor
126129
ServiceName: Service Name
127130
ShareName:
Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
1-
import importlib.util
21
import os
32

3+
from app.translator.tools.utils import execute_module
44
from const import PLATFORMS_PATH
55

66

7-
def init_platforms():
7+
def init_platforms() -> None:
88
for platform in [f for f in os.listdir(PLATFORMS_PATH) if os.path.isdir(os.path.join(PLATFORMS_PATH, f))]:
99
if not platform.startswith("__") and not platform.endswith("__"):
1010
# Platforms __init__.py execution
11-
init_path = f"{PLATFORMS_PATH}/{platform}/__init__.py"
12-
spec = importlib.util.spec_from_file_location("__init__", init_path)
13-
init_module = importlib.util.module_from_spec(spec)
14-
spec.loader.exec_module(init_module)
11+
execute_module(f"{PLATFORMS_PATH}/{platform}/__init__.py")
1512

1613

1714
init_platforms()
Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
UTF8_PAYLOAD_PATTERN = r"UTF8\(payload\)"
22
NUM_VALUE_PATTERN = r"(?P<num_value>\d+(?:\.\d+)*)"
3-
SINGLE_QUOTES_VALUE_PATTERN = r"""'(?P<s_q_value>(?:[:a-zA-Z\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{\}\[\]\s]|'')*)'"""
3+
SINGLE_QUOTES_VALUE_PATTERN = (
4+
r"""'(?P<s_q_value>(?:[:a-zA-Zа-яА-Я\*0-9=+%#\-\/\\|,;_<>`~".$&^@!?\(\)\{\}\[\]\s]|'')*)'""" # noqa: RUF001
5+
)
46
TABLE_PATTERN = r"\s+FROM\s+[a-zA-Z.\-*]+"
57
TABLE_GROUP_PATTERN = r"\s+FROM\s+(?P<table>[a-zA-Z.\-*]+)"

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy