Skip to content

Commit 3c90bed

Browse files
author
Oleksandr Volha
committed
one vendor translations flow
1 parent 8bc0519 commit 3c90bed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+465
-507
lines changed

‎translator/app/translator/core/exceptions/core.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
1-
class NotImplementedException(BaseException):
2-
...
1+
class NotImplementedException(BaseException): ...
32

43

5-
class BasePlatformException(BaseException):
6-
...
4+
class BasePlatformException(BaseException): ...
75

86

97
class StrictPlatformException(BasePlatformException):

‎translator/app/translator/core/exceptions/functions.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
class BaseFunctionException(Exception):
2-
...
1+
class BaseFunctionException(Exception): ...
32

43

54
class InternalFunctionException(Exception):

‎translator/app/translator/core/exceptions/iocs.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
1-
class BaseIOCsException(BaseException):
2-
...
1+
class BaseIOCsException(BaseException): ...
32

43

5-
class IocsLimitExceededException(BaseIOCsException):
6-
...
4+
class IocsLimitExceededException(BaseIOCsException): ...
75

86

97
class EmptyIOCSException(BaseIOCsException):

‎translator/app/translator/core/exceptions/parser.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
class BaseParserException(BaseException):
2-
...
1+
class BaseParserException(BaseException): ...
32

43

54
class TokenizerGeneralException(BaseParserException):

‎translator/app/translator/core/exceptions/render.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
class BaseRenderException(BaseException):
2-
...
1+
class BaseRenderException(BaseException): ...
32

43

54
class UnexpectedLogsourceException(BaseRenderException):
@@ -8,8 +7,7 @@ def __init__(self, platform_name: str, log_source: str):
87
super().__init__(message)
98

109

11-
class FunctionRenderException(BaseRenderException):
12-
...
10+
class FunctionRenderException(BaseRenderException): ...
1311

1412

1513
class UnsupportedRenderMethod(BaseRenderException):

‎translator/app/translator/core/functions.py

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,3 @@
1-
"""
2-
Uncoder IO Commercial Edition License
3-
-----------------------------------------------------------------
4-
Copyright (c) 2023 SOC Prime, Inc.
5-
6-
This file is part of the Uncoder IO Commercial Edition ("CE") and is
7-
licensed under the Uncoder IO Non-Commercial License (the "License");
8-
you may not use this file except in compliance with the License.
9-
You may obtain a copy of the License at
10-
11-
https://github.com/UncoderIO/UncoderIO/blob/main/LICENSE
12-
13-
Unless required by applicable law or agreed to in writing, software
14-
distributed under the License is distributed on an "AS IS" BASIS,
15-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16-
-----------------------------------------------------------------
17-
"""
18-
191
from __future__ import annotations
202

213
from abc import ABC, abstractmethod
@@ -30,7 +12,7 @@
3012
from settings import INIT_FUNCTIONS
3113

3214
if TYPE_CHECKING:
33-
from app.translator.core.render import BaseQueryRender
15+
from app.translator.core.render import PlatformQueryRender
3416

3517

3618
class FunctionParser(ABC):
@@ -72,7 +54,7 @@ def __init__(self):
7254
self._names_map: dict[str, str] = {}
7355

7456
@abstractmethod
75-
def init_search_func_render(self, platform_render: BaseQueryRender) -> None:
57+
def init_search_func_render(self, platform_render: PlatformQueryRender) -> None:
7658
raise NotImplementedError
7759

7860
@cached_property

‎translator/app/translator/core/models/parser_output.py renamed to ‎translator/app/translator/core/models/query_container.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from app.translator.core.mapping import DEFAULT_MAPPING_NAME
88
from app.translator.core.models.field import Field
99
from app.translator.core.models.functions.base import ParsedFunctions
10+
from app.translator.core.tokenizer import TOKEN_TYPE
1011

1112

1213
class MetaInfoContainer:
@@ -27,15 +28,15 @@ def __init__(
2728
status: Optional[str] = None,
2829
false_positives: Optional[list[str]] = None,
2930
source_mapping_ids: Optional[list[str]] = None,
30-
parsed_logsources: Optional[dict] = None
31+
parsed_logsources: Optional[dict] = None,
3132
) -> None:
3233
self.id = id_ or str(uuid.uuid4())
3334
self.title = title or ""
3435
self.description = description or ""
3536
self.author = author or ""
3637
self.date = date or datetime.now().date().strftime("%Y-%m-%d")
37-
self.license = license_ or "DRL 1.1"
3838
self.fields = fields or []
39+
self.license = license_ or "DRL 1.1"
3940
self.severity = severity or SeverityType.low
4041
self.references = references or []
4142
self.tags = tags or []
@@ -47,7 +48,14 @@ def __init__(
4748

4849

4950
@dataclass
50-
class SiemContainer:
51-
query: list
51+
class RawQueryContainer:
52+
query: str
53+
language: str
54+
meta_info: MetaInfoContainer = field(default_factory=MetaInfoContainer)
55+
56+
57+
@dataclass
58+
class TokenizedQueryContainer:
59+
tokens: list[TOKEN_TYPE]
5260
meta_info: MetaInfoContainer
5361
functions: ParsedFunctions = field(default_factory=ParsedFunctions)

‎translator/app/translator/core/parser.py

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,33 +17,35 @@
1717
"""
1818

1919
from abc import ABC, abstractmethod
20+
from typing import Union
2021

2122
from app.translator.core.exceptions.parser import TokenizerGeneralException
2223
from app.translator.core.functions import PlatformFunctions
2324
from app.translator.core.mapping import BasePlatformMappings, SourceMapping
2425
from app.translator.core.models.field import FieldValue
2526
from app.translator.core.models.functions.base import ParsedFunctions
26-
from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer
2727
from app.translator.core.models.platform_details import PlatformDetails
28+
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
2829
from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer
2930

3031

31-
class Parser(ABC):
32-
mappings: BasePlatformMappings = None
33-
tokenizer: QueryTokenizer = None
34-
details: PlatformDetails = None
35-
platform_functions: PlatformFunctions = None
32+
class QueryParser(ABC):
33+
def parse_raw_query(self, text: str, language: str) -> RawQueryContainer:
34+
return RawQueryContainer(query=text, language=language)
3635

3736
@abstractmethod
38-
def _get_meta_info(self, *args, **kwargs) -> MetaInfoContainer:
37+
def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer:
3938
raise NotImplementedError("Abstract method")
4039

41-
@abstractmethod
42-
def parse(self, text: str) -> SiemContainer:
43-
raise NotImplementedError("Abstract method")
40+
41+
class PlatformQueryParser(QueryParser, ABC):
42+
mappings: BasePlatformMappings = None
43+
tokenizer: QueryTokenizer = None
44+
details: PlatformDetails = None
45+
platform_functions: PlatformFunctions = None
4446

4547
def get_tokens_and_source_mappings(
46-
self, query: str, log_sources: dict[str, list[str]]
48+
self, query: str, log_sources: dict[str, Union[str, list[str]]]
4749
) -> tuple[list[TOKEN_TYPE], list[SourceMapping]]:
4850
if not query:
4951
raise TokenizerGeneralException("Can't translate empty query. Please provide more details")

‎translator/app/translator/core/render.py

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19-
from abc import ABC
19+
20+
from abc import ABC, abstractmethod
2021
from collections.abc import Callable
2122
from typing import Optional, Union
2223

@@ -29,11 +30,12 @@
2930
from app.translator.core.functions import PlatformFunctions
3031
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
3132
from app.translator.core.models.field import Field, FieldValue, Keyword
32-
from app.translator.core.models.functions.base import Function, ParsedFunctions
33+
from app.translator.core.models.functions.base import Function
3334
from app.translator.core.models.identifier import Identifier
34-
from app.translator.core.models.parser_output import MetaInfoContainer
3535
from app.translator.core.models.platform_details import PlatformDetails
36+
from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer
3637
from app.translator.core.str_value_manager import StrValueManager
38+
from app.translator.core.tokenizer import TOKEN_TYPE
3739

3840

3941
class BaseQueryFieldValue(ABC):
@@ -99,7 +101,13 @@ def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VAL
99101
raise UnsupportedOperatorException(operator.token_type)
100102

101103

102-
class BaseQueryRender:
104+
class QueryRender(ABC):
105+
@abstractmethod
106+
def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str:
107+
raise NotImplementedError("Abstract method")
108+
109+
110+
class PlatformQueryRender(QueryRender):
103111
mappings: BasePlatformMappings = None
104112
details: PlatformDetails = None
105113
is_strict_mapping = False
@@ -168,9 +176,9 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
168176

169177
return token.token_type
170178

171-
def generate_query(self, query: list[Union[FieldValue, Keyword, Identifier]], source_mapping: SourceMapping) -> str:
179+
def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str:
172180
result_values = []
173-
for token in query:
181+
for token in tokens:
174182
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
175183
return "".join(result_values)
176184

@@ -243,22 +251,33 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp
243251

244252
return source_mappings
245253

246-
def generate(self, query: list, meta_info: MetaInfoContainer, functions: ParsedFunctions) -> str:
254+
def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str:
255+
return self.finalize_query(
256+
prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info
257+
)
258+
259+
def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
247260
queries_map = {}
248-
source_mappings = self._get_source_mappings(meta_info.source_mapping_ids)
261+
source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids)
249262

250263
for source_mapping in source_mappings:
251264
prefix = self.generate_prefix(source_mapping.log_source_signature)
252-
result = self.generate_query(query=query, source_mapping=source_mapping)
265+
result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
253266

254267
finalized_query = self.finalize_query(
255268
prefix=prefix,
256269
query=result,
257-
functions=self.generate_functions(functions.functions, source_mapping),
258-
not_supported_functions=functions.not_supported,
259-
meta_info=meta_info,
270+
functions=self.generate_functions(query_container.functions.functions, source_mapping),
271+
not_supported_functions=query_container.functions.not_supported,
272+
meta_info=query_container.meta_info,
260273
source_mapping=source_mapping,
261274
)
262275
queries_map[source_mapping.source_id] = finalized_query
263276

264277
return self.finalize(queries_map)
278+
279+
def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str:
280+
if isinstance(query_container, RawQueryContainer):
281+
return self._generate_from_raw_query_container(query_container)
282+
283+
return self._generate_from_tokenized_query_container(query_container)

‎translator/app/translator/core/render_cti.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
-----------------------------------------------------------------
1818
"""
1919

20-
2120
from app.translator.core.models.iocs import IocsChunkValue
2221

2322

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy