Skip to content

Commit 92bdb7a

Browse files
alexvolhaalexvolha
authored
Merge pull request #74 from UncoderIO/one-vendor-translations-flow
One vendor translations flow
2 parents 8bc0519 + 8e22eb5 commit 92bdb7a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+433
-425
lines changed

‎translator/app/translator/core/functions.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
from settings import INIT_FUNCTIONS
3131

3232
if TYPE_CHECKING:
33-
from app.translator.core.render import BaseQueryRender
33+
from app.translator.core.render import PlatformQueryRender
3434

3535

3636
class FunctionParser(ABC):
@@ -72,7 +72,7 @@ def __init__(self):
7272
self._names_map: dict[str, str] = {}
7373

7474
@abstractmethod
75-
def init_search_func_render(self, platform_render: BaseQueryRender) -> None:
75+
def init_search_func_render(self, platform_render: PlatformQueryRender) -> None:
7676
raise NotImplementedError
7777

7878
@cached_property

‎translator/app/translator/core/models/parser_output.py renamed to ‎translator/app/translator/core/models/query_container.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from app.translator.core.mapping import DEFAULT_MAPPING_NAME
88
from app.translator.core.models.field import Field
99
from app.translator.core.models.functions.base import ParsedFunctions
10+
from app.translator.core.tokenizer import TOKEN_TYPE
1011

1112

1213
class MetaInfoContainer:
@@ -27,15 +28,15 @@ def __init__(
2728
status: Optional[str] = None,
2829
false_positives: Optional[list[str]] = None,
2930
source_mapping_ids: Optional[list[str]] = None,
30-
parsed_logsources: Optional[dict] = None
31+
parsed_logsources: Optional[dict] = None,
3132
) -> None:
3233
self.id = id_ or str(uuid.uuid4())
3334
self.title = title or ""
3435
self.description = description or ""
3536
self.author = author or ""
3637
self.date = date or datetime.now().date().strftime("%Y-%m-%d")
37-
self.license = license_ or "DRL 1.1"
3838
self.fields = fields or []
39+
self.license = license_ or "DRL 1.1"
3940
self.severity = severity or SeverityType.low
4041
self.references = references or []
4142
self.tags = tags or []
@@ -47,7 +48,14 @@ def __init__(
4748

4849

4950
@dataclass
50-
class SiemContainer:
51-
query: list
51+
class RawQueryContainer:
52+
query: str
53+
language: str
54+
meta_info: MetaInfoContainer = field(default_factory=MetaInfoContainer)
55+
56+
57+
@dataclass
58+
class TokenizedQueryContainer:
59+
tokens: list[TOKEN_TYPE]
5260
meta_info: MetaInfoContainer
5361
functions: ParsedFunctions = field(default_factory=ParsedFunctions)

‎translator/app/translator/core/parser.py

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,33 +17,35 @@
1717
"""
1818

1919
from abc import ABC, abstractmethod
20+
from typing import Union
2021

2122
from app.translator.core.exceptions.parser import TokenizerGeneralException
2223
from app.translator.core.functions import PlatformFunctions
2324
from app.translator.core.mapping import BasePlatformMappings, SourceMapping
2425
from app.translator.core.models.field import FieldValue
2526
from app.translator.core.models.functions.base import ParsedFunctions
26-
from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer
2727
from app.translator.core.models.platform_details import PlatformDetails
28+
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
2829
from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer
2930

3031

31-
class Parser(ABC):
32-
mappings: BasePlatformMappings = None
33-
tokenizer: QueryTokenizer = None
34-
details: PlatformDetails = None
35-
platform_functions: PlatformFunctions = None
32+
class QueryParser(ABC):
33+
def parse_raw_query(self, text: str, language: str) -> RawQueryContainer:
34+
return RawQueryContainer(query=text, language=language)
3635

3736
@abstractmethod
38-
def _get_meta_info(self, *args, **kwargs) -> MetaInfoContainer:
37+
def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer:
3938
raise NotImplementedError("Abstract method")
4039

41-
@abstractmethod
42-
def parse(self, text: str) -> SiemContainer:
43-
raise NotImplementedError("Abstract method")
40+
41+
class PlatformQueryParser(QueryParser, ABC):
42+
mappings: BasePlatformMappings = None
43+
tokenizer: QueryTokenizer = None
44+
details: PlatformDetails = None
45+
platform_functions: PlatformFunctions = None
4446

4547
def get_tokens_and_source_mappings(
46-
self, query: str, log_sources: dict[str, list[str]]
48+
self, query: str, log_sources: dict[str, Union[str, list[str]]]
4749
) -> tuple[list[TOKEN_TYPE], list[SourceMapping]]:
4850
if not query:
4951
raise TokenizerGeneralException("Can't translate empty query. Please provide more details")

‎translator/app/translator/core/render.py

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19-
from abc import ABC
19+
20+
from abc import ABC, abstractmethod
2021
from collections.abc import Callable
2122
from typing import Optional, Union
2223

@@ -29,11 +30,12 @@
2930
from app.translator.core.functions import PlatformFunctions
3031
from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping
3132
from app.translator.core.models.field import Field, FieldValue, Keyword
32-
from app.translator.core.models.functions.base import Function, ParsedFunctions
33+
from app.translator.core.models.functions.base import Function
3334
from app.translator.core.models.identifier import Identifier
34-
from app.translator.core.models.parser_output import MetaInfoContainer
3535
from app.translator.core.models.platform_details import PlatformDetails
36+
from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer
3637
from app.translator.core.str_value_manager import StrValueManager
38+
from app.translator.core.tokenizer import TOKEN_TYPE
3739

3840

3941
class BaseQueryFieldValue(ABC):
@@ -99,7 +101,13 @@ def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VAL
99101
raise UnsupportedOperatorException(operator.token_type)
100102

101103

102-
class BaseQueryRender:
104+
class QueryRender(ABC):
105+
@abstractmethod
106+
def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str:
107+
raise NotImplementedError("Abstract method")
108+
109+
110+
class PlatformQueryRender(QueryRender):
103111
mappings: BasePlatformMappings = None
104112
details: PlatformDetails = None
105113
is_strict_mapping = False
@@ -168,9 +176,9 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
168176

169177
return token.token_type
170178

171-
def generate_query(self, query: list[Union[FieldValue, Keyword, Identifier]], source_mapping: SourceMapping) -> str:
179+
def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str:
172180
result_values = []
173-
for token in query:
181+
for token in tokens:
174182
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
175183
return "".join(result_values)
176184

@@ -243,22 +251,33 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp
243251

244252
return source_mappings
245253

246-
def generate(self, query: list, meta_info: MetaInfoContainer, functions: ParsedFunctions) -> str:
254+
def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str:
255+
return self.finalize_query(
256+
prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info
257+
)
258+
259+
def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
247260
queries_map = {}
248-
source_mappings = self._get_source_mappings(meta_info.source_mapping_ids)
261+
source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids)
249262

250263
for source_mapping in source_mappings:
251264
prefix = self.generate_prefix(source_mapping.log_source_signature)
252-
result = self.generate_query(query=query, source_mapping=source_mapping)
265+
result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
253266

254267
finalized_query = self.finalize_query(
255268
prefix=prefix,
256269
query=result,
257-
functions=self.generate_functions(functions.functions, source_mapping),
258-
not_supported_functions=functions.not_supported,
259-
meta_info=meta_info,
270+
functions=self.generate_functions(query_container.functions.functions, source_mapping),
271+
not_supported_functions=query_container.functions.not_supported,
272+
meta_info=query_container.meta_info,
260273
source_mapping=source_mapping,
261274
)
262275
queries_map[source_mapping.source_id] = finalized_query
263276

264277
return self.finalize(queries_map)
278+
279+
def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str:
280+
if isinstance(query_container, RawQueryContainer):
281+
return self._generate_from_raw_query_container(query_container)
282+
283+
return self._generate_from_tokenized_query_container(query_container)

‎translator/app/translator/core/render_cti.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
-----------------------------------------------------------------
1818
"""
1919

20-
2120
from app.translator.core.models.iocs import IocsChunkValue
2221

2322

‎translator/app/translator/platforms/__init__.py

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
from app.translator.platforms.athena.parsers.athena import AthenaParser
1+
from app.translator.platforms.athena.parsers.athena import AthenaQueryParser
22
from app.translator.platforms.athena.renders.athena import AthenaQueryRender
33
from app.translator.platforms.athena.renders.athena_cti import AthenaCTI
44
from app.translator.platforms.carbonblack.renders.carbonblack_cti import CarbonBlackCTI
5-
from app.translator.platforms.chronicle.parsers.chronicle import ChronicleParser
5+
from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser
66
from app.translator.platforms.chronicle.parsers.chronicle_rule import ChronicleRuleParser
77
from app.translator.platforms.chronicle.renders.chronicle import ChronicleQueryRender
88
from app.translator.platforms.chronicle.renders.chronicle_cti import ChronicleQueryCTI
99
from app.translator.platforms.chronicle.renders.chronicle_rule import ChronicleSecurityRuleRender
10-
from app.translator.platforms.crowdstrike.parsers.crowdstrike import CrowdStrikeParser
10+
from app.translator.platforms.crowdstrike.parsers.crowdstrike import CrowdStrikeQueryParser
1111
from app.translator.platforms.crowdstrike.renders.crowdstrike import CrowdStrikeQueryRender
1212
from app.translator.platforms.crowdstrike.renders.crowdstrike_cti import CrowdStrikeCTI
1313
from app.translator.platforms.elasticsearch.parsers.detection_rule import ElasticSearchRuleParser
14-
from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchParser
14+
from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser
1515
from app.translator.platforms.elasticsearch.renders.detection_rule import ElasticSearchRuleRender
1616
from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender
1717
from app.translator.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender
@@ -20,30 +20,30 @@
2020
from app.translator.platforms.elasticsearch.renders.xpack_watcher import XPackWatcherRuleRender
2121
from app.translator.platforms.fireeye_helix.renders.fireeye_helix_cti import FireeyeHelixCTI
2222
from app.translator.platforms.forti_siem.renders.forti_siem_rule import FortiSiemRuleRender
23-
from app.translator.platforms.graylog.parsers.graylog import GraylogParser
24-
from app.translator.platforms.graylog.renders.graylog import GraylogRender
23+
from app.translator.platforms.graylog.parsers.graylog import GraylogQueryParser
24+
from app.translator.platforms.graylog.renders.graylog import GraylogQueryRender
2525
from app.translator.platforms.graylog.renders.graylog_cti import GraylogCTI
2626
from app.translator.platforms.logpoint.renders.logpoint_cti import LogpointCTI
2727
from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import LogRhythmAxonQueryRender
2828
from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_rule import LogRhythmAxonRuleRender
29-
from app.translator.platforms.logscale.parsers.logscale import LogScaleParser
29+
from app.translator.platforms.logscale.parsers.logscale import LogScaleQueryParser
3030
from app.translator.platforms.logscale.parsers.logscale_alert import LogScaleAlertParser
31-
from app.translator.platforms.logscale.renders.logscale_cti import LogScaleCTI
3231
from app.translator.platforms.logscale.renders.logscale import LogScaleQueryRender
3332
from app.translator.platforms.logscale.renders.logscale_alert import LogScaleAlertRender
33+
from app.translator.platforms.logscale.renders.logscale_cti import LogScaleCTI
3434
from app.translator.platforms.microsoft.parsers.microsoft_defender import MicrosoftDefenderQueryParser
35-
from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftParser
36-
from app.translator.platforms.microsoft.parsers.microsoft_sentinel_rule import MicrosoftRuleParser
35+
from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser
36+
from app.translator.platforms.microsoft.parsers.microsoft_sentinel_rule import MicrosoftSentinelRuleParser
3737
from app.translator.platforms.microsoft.renders.microsoft_defender import MicrosoftDefenderQueryRender
3838
from app.translator.platforms.microsoft.renders.microsoft_defender_cti import MicrosoftDefenderCTI
3939
from app.translator.platforms.microsoft.renders.microsoft_sentinel import MicrosoftSentinelQueryRender
4040
from app.translator.platforms.microsoft.renders.microsoft_sentinel_cti import MicrosoftSentinelCTI
4141
from app.translator.platforms.microsoft.renders.microsoft_sentinel_rule import MicrosoftSentinelRuleRender
42-
from app.translator.platforms.opensearch.parsers.opensearch import OpenSearchParser
42+
from app.translator.platforms.opensearch.parsers.opensearch import OpenSearchQueryParser
4343
from app.translator.platforms.opensearch.renders.opensearch import OpenSearchQueryRender
4444
from app.translator.platforms.opensearch.renders.opensearch_cti import OpenSearchCTI
4545
from app.translator.platforms.opensearch.renders.opensearch_rule import OpenSearchRuleRender
46-
from app.translator.platforms.qradar.parsers.qradar import QradarParser
46+
from app.translator.platforms.qradar.parsers.qradar import QradarQueryParser
4747
from app.translator.platforms.qradar.renders.qradar import QradarQueryRender
4848
from app.translator.platforms.qradar.renders.qradar_cti import QRadarCTI
4949
from app.translator.platforms.qualys.renders.qualys_cti import QualysCTI
@@ -53,7 +53,7 @@
5353
from app.translator.platforms.sigma.parsers.sigma import SigmaParser
5454
from app.translator.platforms.sigma.renders.sigma import SigmaRender
5555
from app.translator.platforms.snowflake.renders.snowflake_cti import SnowflakeCTI
56-
from app.translator.platforms.splunk.parsers.splunk import SplunkParser
56+
from app.translator.platforms.splunk.parsers.splunk import SplunkQueryParser
5757
from app.translator.platforms.splunk.parsers.splunk_alert import SplunkAlertParser
5858
from app.translator.platforms.splunk.renders.splunk import SplunkQueryRender
5959
from app.translator.platforms.splunk.renders.splunk_alert import SplunkAlertRender
@@ -83,28 +83,28 @@
8383
XPackWatcherRuleRender(),
8484
OpenSearchQueryRender(),
8585
OpenSearchRuleRender(),
86-
GraylogRender(),
86+
GraylogQueryRender(),
8787
FortiSiemRuleRender(),
8888
)
8989

9090
__ALL_PARSERS = (
91-
AthenaParser(),
92-
ChronicleParser(),
91+
AthenaQueryParser(),
92+
ChronicleQueryParser(),
9393
ChronicleRuleParser(),
94-
SplunkParser(),
94+
SplunkQueryParser(),
9595
SplunkAlertParser(),
9696
SigmaParser(),
97-
QradarParser(),
98-
MicrosoftParser(),
99-
MicrosoftRuleParser(),
97+
QradarQueryParser(),
98+
MicrosoftSentinelQueryParser(),
99+
MicrosoftSentinelRuleParser(),
100100
MicrosoftDefenderQueryParser(),
101-
CrowdStrikeParser(),
102-
LogScaleParser(),
101+
CrowdStrikeQueryParser(),
102+
LogScaleQueryParser(),
103103
LogScaleAlertParser(),
104-
ElasticSearchParser(),
104+
ElasticSearchQueryParser(),
105105
ElasticSearchRuleParser(),
106-
OpenSearchParser(),
107-
GraylogParser(),
106+
OpenSearchQueryParser(),
107+
GraylogQueryParser(),
108108
)
109109

110110

‎translator/app/translator/platforms/athena/parsers/athena.py

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,39 +19,34 @@
1919
import re
2020
from typing import Optional
2121

22-
from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer
2322
from app.translator.core.models.platform_details import PlatformDetails
24-
from app.translator.core.parser import Parser
23+
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
24+
from app.translator.core.parser import PlatformQueryParser
2525
from app.translator.platforms.athena.const import athena_details
2626
from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings
2727
from app.translator.platforms.athena.tokenizer import AthenaTokenizer
2828

2929

30-
class AthenaParser(Parser):
30+
class AthenaQueryParser(PlatformQueryParser):
3131
details: PlatformDetails = athena_details
3232
mappings: AthenaMappings = athena_mappings
3333
tokenizer = AthenaTokenizer()
3434
query_delimiter_pattern = r"\sFROM\s\S*\sWHERE\s"
3535
table_pattern = r"\sFROM\s(?P<table>[a-zA-Z\.\-\*]+)\sWHERE\s"
3636

37-
@staticmethod
38-
def _get_meta_info(source_mapping_ids: list[str]) -> MetaInfoContainer:
39-
return MetaInfoContainer(source_mapping_ids=source_mapping_ids)
40-
41-
def _parse_query(self, text: str) -> tuple[str, dict[str, Optional[str]]]:
37+
def _parse_query(self, query: str) -> tuple[str, dict[str, Optional[str]]]:
4238
log_source = {"table": None}
43-
if re.search(self.query_delimiter_pattern, text, flags=re.IGNORECASE):
44-
table_search = re.search(self.table_pattern, text)
39+
if re.search(self.query_delimiter_pattern, query, flags=re.IGNORECASE):
40+
table_search = re.search(self.table_pattern, query)
4541
table = table_search.group("table")
4642
log_source["table"] = table
47-
return re.split(self.query_delimiter_pattern, text, flags=re.IGNORECASE)[1], log_source
43+
return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source
4844

49-
return text, log_source
45+
return query, log_source
5046

51-
def parse(self, text: str) -> SiemContainer:
52-
query, log_sources = self._parse_query(text)
47+
def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer:
48+
query, log_sources = self._parse_query(raw_query_container.query)
5349
tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources)
54-
return SiemContainer(
55-
query=tokens,
56-
meta_info=self._get_meta_info([source_mapping.source_id for source_mapping in source_mappings]),
57-
)
50+
meta_info = raw_query_container.meta_info
51+
meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings]
52+
return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info)

‎translator/app/translator/platforms/athena/renders/athena.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,14 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
1920
from typing import Union
2021

2122
from app.translator.const import DEFAULT_VALUE_TYPE
2223
from app.translator.core.exceptions.render import UnsupportedRenderMethod
2324
from app.translator.core.mapping import LogSourceSignature
2425
from app.translator.core.models.platform_details import PlatformDetails
25-
from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender
26+
from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender
2627
from app.translator.platforms.athena.const import athena_details
2728
from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings
2829

@@ -76,7 +77,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00
7677
raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords")
7778

7879

79-
class AthenaQueryRender(BaseQueryRender):
80+
class AthenaQueryRender(PlatformQueryRender):
8081
details: PlatformDetails = athena_details
8182
mappings: AthenaMappings = athena_mappings
8283

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy