diff --git a/translator/app/translator/core/functions.py b/translator/app/translator/core/functions.py index 7c0080ca..02f13657 100644 --- a/translator/app/translator/core/functions.py +++ b/translator/app/translator/core/functions.py @@ -30,7 +30,7 @@ from settings import INIT_FUNCTIONS if TYPE_CHECKING: - from app.translator.core.render import BaseQueryRender + from app.translator.core.render import PlatformQueryRender class FunctionParser(ABC): @@ -72,7 +72,7 @@ def __init__(self): self._names_map: dict[str, str] = {} @abstractmethod - def init_search_func_render(self, platform_render: BaseQueryRender) -> None: + def init_search_func_render(self, platform_render: PlatformQueryRender) -> None: raise NotImplementedError @cached_property diff --git a/translator/app/translator/core/models/parser_output.py b/translator/app/translator/core/models/query_container.py similarity index 85% rename from translator/app/translator/core/models/parser_output.py rename to translator/app/translator/core/models/query_container.py index a17dbc43..b708127c 100644 --- a/translator/app/translator/core/models/parser_output.py +++ b/translator/app/translator/core/models/query_container.py @@ -7,6 +7,7 @@ from app.translator.core.mapping import DEFAULT_MAPPING_NAME from app.translator.core.models.field import Field from app.translator.core.models.functions.base import ParsedFunctions +from app.translator.core.tokenizer import TOKEN_TYPE class MetaInfoContainer: @@ -27,15 +28,15 @@ def __init__( status: Optional[str] = None, false_positives: Optional[list[str]] = None, source_mapping_ids: Optional[list[str]] = None, - parsed_logsources: Optional[dict] = None + parsed_logsources: Optional[dict] = None, ) -> None: self.id = id_ or str(uuid.uuid4()) self.title = title or "" self.description = description or "" self.author = author or "" self.date = date or datetime.now().date().strftime("%Y-%m-%d") - self.license = license_ or "DRL 1.1" self.fields = fields or [] + self.license = license_ or "DRL 1.1" self.severity = severity or SeverityType.low self.references = references or [] self.tags = tags or [] @@ -47,7 +48,14 @@ def __init__( @dataclass -class SiemContainer: - query: list +class RawQueryContainer: + query: str + language: str + meta_info: MetaInfoContainer = field(default_factory=MetaInfoContainer) + + +@dataclass +class TokenizedQueryContainer: + tokens: list[TOKEN_TYPE] meta_info: MetaInfoContainer functions: ParsedFunctions = field(default_factory=ParsedFunctions) diff --git a/translator/app/translator/core/parser.py b/translator/app/translator/core/parser.py index 64097a4a..07450236 100644 --- a/translator/app/translator/core/parser.py +++ b/translator/app/translator/core/parser.py @@ -17,33 +17,35 @@ """ from abc import ABC, abstractmethod +from typing import Union from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping from app.translator.core.models.field import FieldValue from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer -class Parser(ABC): - mappings: BasePlatformMappings = None - tokenizer: QueryTokenizer = None - details: PlatformDetails = None - platform_functions: PlatformFunctions = None +class QueryParser(ABC): + def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: + return RawQueryContainer(query=text, language=language) @abstractmethod - def _get_meta_info(self, *args, **kwargs) -> MetaInfoContainer: + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: raise NotImplementedError("Abstract method") - @abstractmethod - def parse(self, text: str) -> SiemContainer: - raise NotImplementedError("Abstract method") + +class PlatformQueryParser(QueryParser, ABC): + mappings: BasePlatformMappings = None + tokenizer: QueryTokenizer = None + details: PlatformDetails = None + platform_functions: PlatformFunctions = None def get_tokens_and_source_mappings( - self, query: str, log_sources: dict[str, list[str]] + self, query: str, log_sources: dict[str, Union[str, list[str]]] ) -> tuple[list[TOKEN_TYPE], list[SourceMapping]]: if not query: raise TokenizerGeneralException("Can't translate empty query. Please provide more details") diff --git a/translator/app/translator/core/render.py b/translator/app/translator/core/render.py index c26416bf..039a8d53 100644 --- a/translator/app/translator/core/render.py +++ b/translator/app/translator/core/render.py @@ -16,7 +16,8 @@ limitations under the License. ----------------------------------------------------------------- """ -from abc import ABC + +from abc import ABC, abstractmethod from collections.abc import Callable from typing import Optional, Union @@ -29,11 +30,12 @@ from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping from app.translator.core.models.field import Field, FieldValue, Keyword -from app.translator.core.models.functions.base import Function, ParsedFunctions +from app.translator.core.models.functions.base import Function from app.translator.core.models.identifier import Identifier -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.str_value_manager import StrValueManager +from app.translator.core.tokenizer import TOKEN_TYPE class BaseQueryFieldValue(ABC): @@ -99,7 +101,13 @@ def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VAL raise UnsupportedOperatorException(operator.token_type) -class BaseQueryRender: +class QueryRender(ABC): + @abstractmethod + def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: + raise NotImplementedError("Abstract method") + + +class PlatformQueryRender(QueryRender): mappings: BasePlatformMappings = None details: PlatformDetails = None is_strict_mapping = False @@ -168,9 +176,9 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp return token.token_type - def generate_query(self, query: list[Union[FieldValue, Keyword, Identifier]], source_mapping: SourceMapping) -> str: + def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str: result_values = [] - for token in query: + for token in tokens: result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) return "".join(result_values) @@ -243,22 +251,33 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp return source_mappings - def generate(self, query: list, meta_info: MetaInfoContainer, functions: ParsedFunctions) -> str: + def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + return self.finalize_query( + prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info + ) + + def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} - source_mappings = self._get_source_mappings(meta_info.source_mapping_ids) + source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) for source_mapping in source_mappings: prefix = self.generate_prefix(source_mapping.log_source_signature) - result = self.generate_query(query=query, source_mapping=source_mapping) + result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) finalized_query = self.finalize_query( prefix=prefix, query=result, - functions=self.generate_functions(functions.functions, source_mapping), - not_supported_functions=functions.not_supported, - meta_info=meta_info, + functions=self.generate_functions(query_container.functions.functions, source_mapping), + not_supported_functions=query_container.functions.not_supported, + meta_info=query_container.meta_info, source_mapping=source_mapping, ) queries_map[source_mapping.source_id] = finalized_query return self.finalize(queries_map) + + def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: + if isinstance(query_container, RawQueryContainer): + return self._generate_from_raw_query_container(query_container) + + return self._generate_from_tokenized_query_container(query_container) diff --git a/translator/app/translator/core/render_cti.py b/translator/app/translator/core/render_cti.py index f07d5c15..30509a71 100644 --- a/translator/app/translator/core/render_cti.py +++ b/translator/app/translator/core/render_cti.py @@ -17,7 +17,6 @@ ----------------------------------------------------------------- """ - from app.translator.core.models.iocs import IocsChunkValue diff --git a/translator/app/translator/platforms/__init__.py b/translator/app/translator/platforms/__init__.py index af92d5be..4a30fd41 100644 --- a/translator/app/translator/platforms/__init__.py +++ b/translator/app/translator/platforms/__init__.py @@ -1,17 +1,17 @@ -from app.translator.platforms.athena.parsers.athena import AthenaParser +from app.translator.platforms.athena.parsers.athena import AthenaQueryParser from app.translator.platforms.athena.renders.athena import AthenaQueryRender from app.translator.platforms.athena.renders.athena_cti import AthenaCTI from app.translator.platforms.carbonblack.renders.carbonblack_cti import CarbonBlackCTI -from app.translator.platforms.chronicle.parsers.chronicle import ChronicleParser +from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser from app.translator.platforms.chronicle.parsers.chronicle_rule import ChronicleRuleParser from app.translator.platforms.chronicle.renders.chronicle import ChronicleQueryRender from app.translator.platforms.chronicle.renders.chronicle_cti import ChronicleQueryCTI from app.translator.platforms.chronicle.renders.chronicle_rule import ChronicleSecurityRuleRender -from app.translator.platforms.crowdstrike.parsers.crowdstrike import CrowdStrikeParser +from app.translator.platforms.crowdstrike.parsers.crowdstrike import CrowdStrikeQueryParser from app.translator.platforms.crowdstrike.renders.crowdstrike import CrowdStrikeQueryRender from app.translator.platforms.crowdstrike.renders.crowdstrike_cti import CrowdStrikeCTI from app.translator.platforms.elasticsearch.parsers.detection_rule import ElasticSearchRuleParser -from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchParser +from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser from app.translator.platforms.elasticsearch.renders.detection_rule import ElasticSearchRuleRender from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender from app.translator.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender @@ -20,30 +20,30 @@ from app.translator.platforms.elasticsearch.renders.xpack_watcher import XPackWatcherRuleRender from app.translator.platforms.fireeye_helix.renders.fireeye_helix_cti import FireeyeHelixCTI from app.translator.platforms.forti_siem.renders.forti_siem_rule import FortiSiemRuleRender -from app.translator.platforms.graylog.parsers.graylog import GraylogParser -from app.translator.platforms.graylog.renders.graylog import GraylogRender +from app.translator.platforms.graylog.parsers.graylog import GraylogQueryParser +from app.translator.platforms.graylog.renders.graylog import GraylogQueryRender from app.translator.platforms.graylog.renders.graylog_cti import GraylogCTI from app.translator.platforms.logpoint.renders.logpoint_cti import LogpointCTI from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import LogRhythmAxonQueryRender from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_rule import LogRhythmAxonRuleRender -from app.translator.platforms.logscale.parsers.logscale import LogScaleParser +from app.translator.platforms.logscale.parsers.logscale import LogScaleQueryParser from app.translator.platforms.logscale.parsers.logscale_alert import LogScaleAlertParser -from app.translator.platforms.logscale.renders.logscale_cti import LogScaleCTI from app.translator.platforms.logscale.renders.logscale import LogScaleQueryRender from app.translator.platforms.logscale.renders.logscale_alert import LogScaleAlertRender +from app.translator.platforms.logscale.renders.logscale_cti import LogScaleCTI from app.translator.platforms.microsoft.parsers.microsoft_defender import MicrosoftDefenderQueryParser -from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftParser -from app.translator.platforms.microsoft.parsers.microsoft_sentinel_rule import MicrosoftRuleParser +from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser +from app.translator.platforms.microsoft.parsers.microsoft_sentinel_rule import MicrosoftSentinelRuleParser from app.translator.platforms.microsoft.renders.microsoft_defender import MicrosoftDefenderQueryRender from app.translator.platforms.microsoft.renders.microsoft_defender_cti import MicrosoftDefenderCTI from app.translator.platforms.microsoft.renders.microsoft_sentinel import MicrosoftSentinelQueryRender from app.translator.platforms.microsoft.renders.microsoft_sentinel_cti import MicrosoftSentinelCTI from app.translator.platforms.microsoft.renders.microsoft_sentinel_rule import MicrosoftSentinelRuleRender -from app.translator.platforms.opensearch.parsers.opensearch import OpenSearchParser +from app.translator.platforms.opensearch.parsers.opensearch import OpenSearchQueryParser from app.translator.platforms.opensearch.renders.opensearch import OpenSearchQueryRender from app.translator.platforms.opensearch.renders.opensearch_cti import OpenSearchCTI from app.translator.platforms.opensearch.renders.opensearch_rule import OpenSearchRuleRender -from app.translator.platforms.qradar.parsers.qradar import QradarParser +from app.translator.platforms.qradar.parsers.qradar import QradarQueryParser from app.translator.platforms.qradar.renders.qradar import QradarQueryRender from app.translator.platforms.qradar.renders.qradar_cti import QRadarCTI from app.translator.platforms.qualys.renders.qualys_cti import QualysCTI @@ -53,7 +53,7 @@ from app.translator.platforms.sigma.parsers.sigma import SigmaParser from app.translator.platforms.sigma.renders.sigma import SigmaRender from app.translator.platforms.snowflake.renders.snowflake_cti import SnowflakeCTI -from app.translator.platforms.splunk.parsers.splunk import SplunkParser +from app.translator.platforms.splunk.parsers.splunk import SplunkQueryParser from app.translator.platforms.splunk.parsers.splunk_alert import SplunkAlertParser from app.translator.platforms.splunk.renders.splunk import SplunkQueryRender from app.translator.platforms.splunk.renders.splunk_alert import SplunkAlertRender @@ -83,28 +83,28 @@ XPackWatcherRuleRender(), OpenSearchQueryRender(), OpenSearchRuleRender(), - GraylogRender(), + GraylogQueryRender(), FortiSiemRuleRender(), ) __ALL_PARSERS = ( - AthenaParser(), - ChronicleParser(), + AthenaQueryParser(), + ChronicleQueryParser(), ChronicleRuleParser(), - SplunkParser(), + SplunkQueryParser(), SplunkAlertParser(), SigmaParser(), - QradarParser(), - MicrosoftParser(), - MicrosoftRuleParser(), + QradarQueryParser(), + MicrosoftSentinelQueryParser(), + MicrosoftSentinelRuleParser(), MicrosoftDefenderQueryParser(), - CrowdStrikeParser(), - LogScaleParser(), + CrowdStrikeQueryParser(), + LogScaleQueryParser(), LogScaleAlertParser(), - ElasticSearchParser(), + ElasticSearchQueryParser(), ElasticSearchRuleParser(), - OpenSearchParser(), - GraylogParser(), + OpenSearchQueryParser(), + GraylogQueryParser(), ) diff --git a/translator/app/translator/platforms/athena/parsers/athena.py b/translator/app/translator/platforms/athena/parsers/athena.py index 63fa31ed..fe54e1f0 100644 --- a/translator/app/translator/platforms/athena/parsers/athena.py +++ b/translator/app/translator/platforms/athena/parsers/athena.py @@ -19,39 +19,34 @@ import re from typing import Optional -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.parser import Parser +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.athena.const import athena_details from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings from app.translator.platforms.athena.tokenizer import AthenaTokenizer -class AthenaParser(Parser): +class AthenaQueryParser(PlatformQueryParser): details: PlatformDetails = athena_details mappings: AthenaMappings = athena_mappings tokenizer = AthenaTokenizer() query_delimiter_pattern = r"\sFROM\s\S*\sWHERE\s" table_pattern = r"\sFROM\s(?P[a-zA-Z\.\-\*]+)\sWHERE\s" - @staticmethod - def _get_meta_info(source_mapping_ids: list[str]) -> MetaInfoContainer: - return MetaInfoContainer(source_mapping_ids=source_mapping_ids) - - def _parse_query(self, text: str) -> tuple[str, dict[str, Optional[str]]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, Optional[str]]]: log_source = {"table": None} - if re.search(self.query_delimiter_pattern, text, flags=re.IGNORECASE): - table_search = re.search(self.table_pattern, text) + if re.search(self.query_delimiter_pattern, query, flags=re.IGNORECASE): + table_search = re.search(self.table_pattern, query) table = table_search.group("table") log_source["table"] = table - return re.split(self.query_delimiter_pattern, text, flags=re.IGNORECASE)[1], log_source + return re.split(self.query_delimiter_pattern, query, flags=re.IGNORECASE)[1], log_source - return text, log_source + return query, log_source - def parse(self, text: str) -> SiemContainer: - query, log_sources = self._parse_query(text) + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + query, log_sources = self._parse_query(raw_query_container.query) tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info([source_mapping.source_id for source_mapping in source_mappings]), - ) + meta_info = raw_query_container.meta_info + meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] + return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) diff --git a/translator/app/translator/platforms/athena/renders/athena.py b/translator/app/translator/platforms/athena/renders/athena.py index d32e5f8a..0efe4309 100644 --- a/translator/app/translator/platforms/athena/renders/athena.py +++ b/translator/app/translator/platforms/athena/renders/athena.py @@ -16,13 +16,14 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.platforms.athena.const import athena_details from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings @@ -76,7 +77,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords") -class AthenaQueryRender(BaseQueryRender): +class AthenaQueryRender(PlatformQueryRender): details: PlatformDetails = athena_details mappings: AthenaMappings = athena_mappings diff --git a/translator/app/translator/platforms/base/lucene/parsers/lucene.py b/translator/app/translator/platforms/base/lucene/parsers/lucene.py index 886f11e1..ca4caf09 100644 --- a/translator/app/translator/platforms/base/lucene/parsers/lucene.py +++ b/translator/app/translator/platforms/base/lucene/parsers/lucene.py @@ -18,18 +18,18 @@ import re -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer -from app.translator.core.parser import Parser +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.lucene.tokenizer import LuceneTokenizer -class LuceneParser(Parser): +class LuceneQueryParser(PlatformQueryParser): tokenizer = LuceneTokenizer() log_source_pattern = r"___source_type___\s*(?:[:=])\s*(?:\"?(?P[%a-zA-Z_*:0-9\-/]+)\"|(?P[%a-zA-Z_*:0-9\-/]+))(?:\s+(?:and|or)\s+|\s+)?" # noqa: E501 log_source_key_types = ("index", "event\.category") - def _parse_log_sources(self, query: str) -> tuple[str, dict[str, list[str]]]: + def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: log_sources = {} for source_type in self.log_source_key_types: pattern = self.log_source_pattern.replace("___source_type___", source_type) @@ -43,17 +43,9 @@ def _parse_log_sources(self, query: str) -> tuple[str, dict[str, list[str]]]: return query, log_sources - @staticmethod - def _get_meta_info(source_mapping_ids: list[str], meta_info: dict) -> MetaInfoContainer: # noqa: ARG004 - return MetaInfoContainer(source_mapping_ids=source_mapping_ids) - - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]]]: - return self._parse_log_sources(query) - - def parse(self, text: str) -> SiemContainer: - query, log_sources = self._parse_query(text) + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + query, log_sources = self._parse_query(raw_query_container.query) tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info([source_mapping.source_id for source_mapping in source_mappings], {}), - ) + meta_info = raw_query_container.meta_info + meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] + return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) diff --git a/translator/app/translator/platforms/base/lucene/renders/lucene.py b/translator/app/translator/platforms/base/lucene/renders/lucene.py index 82450b18..f06a9c7a 100644 --- a/translator/app/translator/platforms/base/lucene/renders/lucene.py +++ b/translator/app/translator/platforms/base/lucene/renders/lucene.py @@ -16,11 +16,12 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.lucene.mapping import LuceneLogSourceSignature from app.translator.platforms.base.lucene.str_value_manager import lucene_str_value_manager @@ -125,7 +126,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"*{self._pre_process_value(field, value)}*" -class LuceneQueryRender(BaseQueryRender): +class LuceneQueryRender(PlatformQueryRender): or_token = "OR" and_token = "AND" not_token = "NOT" diff --git a/translator/app/translator/platforms/base/lucene/tokenizer.py b/translator/app/translator/platforms/base/lucene/tokenizer.py index b5bde101..0a0f902d 100644 --- a/translator/app/translator/platforms/base/lucene/tokenizer.py +++ b/translator/app/translator/platforms/base/lucene/tokenizer.py @@ -15,6 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ + import re from typing import ClassVar, Union diff --git a/translator/app/translator/platforms/base/spl/parsers/spl.py b/translator/app/translator/platforms/base/spl/parsers/spl.py index 21d055bb..9270defc 100644 --- a/translator/app/translator/platforms/base/spl/parsers/spl.py +++ b/translator/app/translator/platforms/base/spl/parsers/spl.py @@ -17,15 +17,14 @@ """ import re -from typing import Optional from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer -from app.translator.core.parser import Parser +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.base.spl.tokenizer import SplTokenizer -class SplParser(Parser): +class SplQueryParser(PlatformQueryParser): log_source_pattern = r"^___source_type___\s*=\s*(?:\"(?P[%a-zA-Z_*:0-9\-/]+)\"|(?P[%a-zA-Z_*:0-9\-/]+))(?:\s+(?:and|or)\s+|\s+)?" # noqa: E501 log_source_key_types = ("index", "source", "sourcetype", "sourcecategory") @@ -47,19 +46,16 @@ def _parse_log_sources(self, query: str) -> tuple[dict[str, list[str]], str]: return log_sources, query - def _parse_query(self, query: str) -> tuple[dict[str, list[str]], ParsedFunctions, str]: + def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFunctions]: query = query.strip() log_sources, query = self._parse_log_sources(query) query, functions = self.platform_functions.parse(query) - return log_sources, functions, query + return query, log_sources, functions - @staticmethod - def _get_meta_info(source_mapping_ids: list[str], meta_info: Optional[dict]) -> MetaInfoContainer: # noqa: ARG004 - return MetaInfoContainer(source_mapping_ids=source_mapping_ids) - - def parse(self, text: str) -> SiemContainer: - log_sources, functions, query = self._parse_query(text) + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + query, log_sources, functions = self._parse_query(raw_query_container.query) tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) - meta_info = self._get_meta_info([source_mapping.source_id for source_mapping in source_mappings], {}) - return SiemContainer(query=tokens, meta_info=meta_info, functions=functions) + meta_info = raw_query_container.meta_info + meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] + return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) diff --git a/translator/app/translator/platforms/base/spl/renders/spl.py b/translator/app/translator/platforms/base/spl/renders/spl.py index c449439f..d5b71ff1 100644 --- a/translator/app/translator/platforms/base/spl/renders/spl.py +++ b/translator/app/translator/platforms/base/spl/renders/spl.py @@ -16,11 +16,12 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod -from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.platforms.base.spl.escape_manager import spl_escape_manager @@ -73,7 +74,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: raise UnsupportedRenderMethod(platform_name=self.details.name, method="Regex Expression") -class SplQueryRender(BaseQueryRender): +class SplQueryRender(PlatformQueryRender): or_token = "OR" and_token = "AND" not_token = "NOT" diff --git a/translator/app/translator/platforms/chronicle/parsers/chronicle.py b/translator/app/translator/platforms/chronicle/parsers/chronicle.py index e64c07b2..30767534 100644 --- a/translator/app/translator/platforms/chronicle/parsers/chronicle.py +++ b/translator/app/translator/platforms/chronicle/parsers/chronicle.py @@ -16,26 +16,21 @@ ----------------------------------------------------------------- """ - -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.parser import Parser +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.chronicle.const import chronicle_query_details from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings from app.translator.platforms.chronicle.tokenizer import ChronicleQueryTokenizer -class ChronicleParser(Parser): +class ChronicleQueryParser(PlatformQueryParser): mappings: ChronicleMappings = chronicle_mappings tokenizer: ChronicleQueryTokenizer = ChronicleQueryTokenizer() details: PlatformDetails = chronicle_query_details - def _get_meta_info(self, source_mapping_ids: list[str]) -> MetaInfoContainer: - return MetaInfoContainer(source_mapping_ids=source_mapping_ids) - - def parse(self, text: str) -> SiemContainer: - tokens, source_mappings = self.get_tokens_and_source_mappings(text, {}) - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info([source_mapping.source_id for source_mapping in source_mappings]), - ) + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + tokens, source_mappings = self.get_tokens_and_source_mappings(raw_query_container.query, {}) + meta_info = raw_query_container.meta_info + meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] + return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) diff --git a/translator/app/translator/platforms/chronicle/parsers/chronicle_rule.py b/translator/app/translator/platforms/chronicle/parsers/chronicle_rule.py index 635d16ac..88319fc9 100644 --- a/translator/app/translator/platforms/chronicle/parsers/chronicle_rule.py +++ b/translator/app/translator/platforms/chronicle/parsers/chronicle_rule.py @@ -19,80 +19,70 @@ import re from app.translator.core.exceptions.parser import TokenizerGeneralException -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.parser import Parser +from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.platforms.chronicle.const import chronicle_rule_details from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings +from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser from app.translator.platforms.chronicle.tokenizer import ChronicleRuleTokenizer -class ChronicleRuleParser(Parser): +class ChronicleRuleParser(ChronicleQueryParser): details: PlatformDetails = chronicle_rule_details rule_name_pattern = "rule\s(?P[a-z0-9_]+)\s{" meta_info_pattern = "meta:\n(?P[a-zA-Z0-9_\\\.*,>–<—~#$’`:;%+^\|?!@\s\"/=\-&'\(\)\[\]]+)\n\s+events:" # noqa: RUF001 - rule_pattern = "events:\n\s*(?P[a-zA-Z\w0-9_%{}\|\.,!#^><:~\s\"\/=+?\-–&;$()`\*@\[\]'\\\]+)\n\s+condition:" # noqa: RUF001 + rule_pattern = "events:\n\s*(?P[a-zA-Z\w0-9_%{}\|\.,!#^><:~\s\"\/=+?\-–&;$()`\*@\[\]'\\\]+)\n\s+condition:" # noqa: RUF001 event_name_pattern = "condition:\n\s*(?P\$[a-zA-Z_0-9]+)\n" mappings: ChronicleMappings = chronicle_mappings tokenizer = ChronicleRuleTokenizer() - def __parse_rule(self, rule: str) -> dict: - rule_name_search = re.search(self.rule_name_pattern, rule) - if rule_name_search is None: - raise TokenizerGeneralException(error="Field couldn't be found in rule.") + def __parse_rule(self, rule: str) -> tuple[str, str, str]: + if (rule_name_search := re.search(self.rule_name_pattern, rule)) is None: + raise TokenizerGeneralException(error="Rule name couldn't be found in rule.") rule_name = rule_name_search.group("rule_name") - meta_info_search = re.search(self.meta_info_pattern, rule) - if meta_info_search is None: + if (meta_info_search := re.search(self.meta_info_pattern, rule)) is None: raise TokenizerGeneralException(error="Rule meta info couldn't be found in rule.") meta_info = meta_info_search.group("meta_info") - query_search = re.search(self.rule_pattern, rule) - if query_search is None: + if (query_search := re.search(self.rule_pattern, rule)) is None: raise TokenizerGeneralException(error="Query couldn't be found in rule.") - query = query_search.group("rule") + query = query_search.group("query") - event_name_search = re.search(self.event_name_pattern, rule) - if query_search is None: + if (event_name_search := re.search(self.event_name_pattern, rule)) is None: raise TokenizerGeneralException(error="Event name couldn't be found in rule.") event_name = event_name_search.group("event_name") query = query.replace(f"{event_name}.", "") - return {"query": query.strip(" ").strip("\n"), "rule_name": rule_name, "meta_info": meta_info} + return query.strip(" ").strip("\n"), rule_name, meta_info @staticmethod def __prepare_title(name: str) -> str: return " ".join(name.split("_")).title() - def _get_meta_info(self, rule_name: str, source_mapping_ids: list[str], meta_info: str) -> MetaInfoContainer: + @staticmethod + def __parse_meta_info(meta_info_str: str) -> tuple[str, list[str], list[str]]: references = tags = [] description = None - for info in meta_info.strip(" ").strip("\n").split("\n"): + for info in meta_info_str.strip(" ").strip("\n").split("\n"): key, value = info.split(" = ") key = key.strip(" ") - if key == "reference": - references = [value.strip(" ").strip('"')] - elif key == "description": + if key == "description": description = value.strip(" ") + elif key == "reference": + references = [value.strip(" ").strip('"')] elif key == "tags": tags = [i.strip(" ").strip('"') for i in value.split(",")] - return MetaInfoContainer( - title=self.__prepare_title(rule_name), - source_mapping_ids=source_mapping_ids, - references=references, - tags=tags, - description=description, - ) + return description, references, tags - def parse(self, text: str) -> SiemContainer: - parsed_rule = self.__parse_rule(text) - tokens, source_mappings = self.get_tokens_and_source_mappings(parsed_rule.get("query"), {}) - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info( - rule_name=parsed_rule.get("rule_name"), - meta_info=parsed_rule.get("meta_info"), - source_mapping_ids=[source_mapping.source_id for source_mapping in source_mappings], + def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: + query, rule_name, meta_info_str = self.__parse_rule(text) + description, references, tags = self.__parse_meta_info(meta_info_str) + return RawQueryContainer( + query=query, + language=language, + meta_info=MetaInfoContainer( + title=self.__prepare_title(rule_name), description=description, references=references, tags=tags ), ) diff --git a/translator/app/translator/platforms/chronicle/renders/chronicle.py b/translator/app/translator/platforms/chronicle/renders/chronicle.py index b4a5e993..fdc02aa0 100644 --- a/translator/app/translator/platforms/chronicle/renders/chronicle.py +++ b/translator/app/translator/platforms/chronicle/renders/chronicle.py @@ -16,13 +16,14 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.platforms.chronicle.const import chronicle_query_details from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings @@ -96,7 +97,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords") -class ChronicleQueryRender(BaseQueryRender): +class ChronicleQueryRender(PlatformQueryRender): details: PlatformDetails = chronicle_query_details mappings: ChronicleMappings = chronicle_mappings diff --git a/translator/app/translator/platforms/chronicle/renders/chronicle_rule.py b/translator/app/translator/platforms/chronicle/renders/chronicle_rule.py index 5967de4a..f81005c3 100644 --- a/translator/app/translator/platforms/chronicle/renders/chronicle_rule.py +++ b/translator/app/translator/platforms/chronicle/renders/chronicle_rule.py @@ -22,8 +22,8 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValue, ChronicleQueryRender diff --git a/translator/app/translator/platforms/crowdstrike/parsers/crowdstrike.py b/translator/app/translator/platforms/crowdstrike/parsers/crowdstrike.py index fc34b670..08aabb12 100644 --- a/translator/app/translator/platforms/crowdstrike/parsers/crowdstrike.py +++ b/translator/app/translator/platforms/crowdstrike/parsers/crowdstrike.py @@ -15,14 +15,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ + from app.translator.core.models.platform_details import PlatformDetails -from app.translator.platforms.base.spl.parsers.spl import SplParser +from app.translator.platforms.base.spl.parsers.spl import SplQueryParser from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings -class CrowdStrikeParser(SplParser): +class CrowdStrikeQueryParser(SplQueryParser): details: PlatformDetails = crowdstrike_query_details log_source_pattern = r"___source_type___\s*=\s*(?:\"(?P[%a-zA-Z_*:0-9\-/]+)\"|(?P[%a-zA-Z_*:0-9\-/]+))(?:\s+(?:and|or)\s+|\s+)?" # noqa: E501 diff --git a/translator/app/translator/platforms/crowdstrike/renders/crowdstrike.py b/translator/app/translator/platforms/crowdstrike/renders/crowdstrike.py index 5cda71f8..6c1c2e6c 100644 --- a/translator/app/translator/platforms/crowdstrike/renders/crowdstrike.py +++ b/translator/app/translator/platforms/crowdstrike/renders/crowdstrike.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from app.translator.core.models.platform_details import PlatformDetails from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender from app.translator.platforms.crowdstrike.const import crowdstrike_query_details diff --git a/translator/app/translator/platforms/elasticsearch/const.py b/translator/app/translator/platforms/elasticsearch/const.py index 4b811ed2..24c15a0b 100644 --- a/translator/app/translator/platforms/elasticsearch/const.py +++ b/translator/app/translator/platforms/elasticsearch/const.py @@ -3,15 +3,29 @@ PLATFORM_DETAILS = {"group_id": "elk stack", "group_name": "Elastic Stack", "alt_platform_name": "ECS"} +_ELASTIC_LUCENE_QUERY = "elastic-lucene-query" +_ELASTIC_LUCENE_RULE = "elastic-lucene-rule" +_ELASTIC_KIBANA_RULE = "elastic-kibana-rule" +_ELASTALERT_LUCENE_RULE = "elastalert-lucene-rule" +_ELASTIC_WATCHER_RULE = "elastic-watcher-rule" + +ELASTIC_QUERY_TYPES = { + _ELASTIC_LUCENE_QUERY, + _ELASTIC_LUCENE_RULE, + _ELASTIC_KIBANA_RULE, + _ELASTALERT_LUCENE_RULE, + _ELASTIC_WATCHER_RULE, +} + ELASTICSEARCH_LUCENE_QUERY_DETAILS = { - "siem_type": "elastic-lucene-query", + "siem_type": _ELASTIC_LUCENE_QUERY, "name": "Elasticsearch Query", "platform_name": "Query (Lucene)", **PLATFORM_DETAILS, } ELASTICSEARCH_RULE_DETAILS = { - "siem_type": "elastic-lucene-rule", + "siem_type": _ELASTIC_LUCENE_RULE, "name": "Elastic Rule", "platform_name": "Detection Rule (Lucene)", "first_choice": 0, @@ -19,7 +33,7 @@ } KIBANA_DETAILS = { - "siem_type": "elastic-kibana-rule", + "siem_type": _ELASTIC_KIBANA_RULE, "name": "Elastic Kibana Saved Search", "platform_name": "Kibana SavedSearch (JSON)", "first_choice": 0, @@ -27,7 +41,7 @@ } ELASTALERT_DETAILS = { - "siem_type": "elastalert-lucene-rule", + "siem_type": _ELASTALERT_LUCENE_RULE, "name": "ElastAlert", "platform_name": "Alert (Lucene)", "group_name": "ElastAlert", @@ -35,7 +49,7 @@ } XPACK_WATCHER_DETAILS = { - "siem_type": "elastic-watcher-rule", + "siem_type": _ELASTIC_WATCHER_RULE, "name": "Elastic Watcher", "platform_name": "Rule (Watcher)", "first_choice": 0, diff --git a/translator/app/translator/platforms/elasticsearch/parsers/detection_rule.py b/translator/app/translator/platforms/elasticsearch/parsers/detection_rule.py index 316ab577..229cba4b 100644 --- a/translator/app/translator/platforms/elasticsearch/parsers/detection_rule.py +++ b/translator/app/translator/platforms/elasticsearch/parsers/detection_rule.py @@ -16,37 +16,20 @@ ----------------------------------------------------------------- """ - from app.translator.core.mixins.rule import JsonRuleMixin -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.platforms.elasticsearch.const import elasticsearch_rule_details -from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchParser +from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser -class ElasticSearchRuleParser(ElasticSearchParser, JsonRuleMixin): +class ElasticSearchRuleParser(ElasticSearchQueryParser, JsonRuleMixin): details: PlatformDetails = elasticsearch_rule_details - def _parse_rule(self, text: str) -> dict: + def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: rule = self.load_rule(text=text) - query = rule["query"] - description = rule["description"] - name = rule["name"] - query, logsources = self._parse_log_sources(query) - return {"query": query, "title": name, "description": description, "logsources": logsources} - - @staticmethod - def _get_meta_info(source_mapping_ids: list[str], meta_info: dict) -> MetaInfoContainer: - return MetaInfoContainer( - title=meta_info["title"], description=meta_info["description"], source_mapping_ids=source_mapping_ids - ) - - def parse(self, text: str) -> SiemContainer: - rule = self._parse_rule(text) - tokens, source_mappings = self.get_tokens_and_source_mappings(rule.get("query"), rule.get("log_sources", {})) - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info( - source_mapping_ids=[source_mapping.source_id for source_mapping in source_mappings], meta_info=rule - ), + return RawQueryContainer( + query=rule["query"], + language=language, + meta_info=MetaInfoContainer(title=rule["name"], description=rule["description"]), ) diff --git a/translator/app/translator/platforms/elasticsearch/parsers/elasticsearch.py b/translator/app/translator/platforms/elasticsearch/parsers/elasticsearch.py index e51ebb32..88ae8ca0 100644 --- a/translator/app/translator/platforms/elasticsearch/parsers/elasticsearch.py +++ b/translator/app/translator/platforms/elasticsearch/parsers/elasticsearch.py @@ -17,11 +17,11 @@ """ from app.translator.core.models.platform_details import PlatformDetails -from app.translator.platforms.base.lucene.parsers.lucene import LuceneParser +from app.translator.platforms.base.lucene.parsers.lucene import LuceneQueryParser from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -class ElasticSearchParser(LuceneParser): +class ElasticSearchQueryParser(LuceneQueryParser): details: PlatformDetails = elasticsearch_lucene_query_details mappings: ElasticSearchMappings = elasticsearch_mappings diff --git a/translator/app/translator/platforms/elasticsearch/renders/detection_rule.py b/translator/app/translator/platforms/elasticsearch/renders/detection_rule.py index 7211f9b9..ad8e7a94 100644 --- a/translator/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/translator/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -24,8 +24,8 @@ from app.translator.core.mapping import SourceMapping from app.translator.core.mitre import MitreConfig -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_DETECTION_RULE, elasticsearch_rule_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( diff --git a/translator/app/translator/platforms/elasticsearch/renders/elast_alert.py b/translator/app/translator/platforms/elasticsearch/renders/elast_alert.py index 15c86efd..825c6713 100644 --- a/translator/app/translator/platforms/elasticsearch/renders/elast_alert.py +++ b/translator/app/translator/platforms/elasticsearch/renders/elast_alert.py @@ -16,12 +16,13 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Optional from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.elasticsearch.const import ELASTICSEARCH_ALERT, elastalert_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( diff --git a/translator/app/translator/platforms/elasticsearch/renders/kibana.py b/translator/app/translator/platforms/elasticsearch/renders/kibana.py index af5b8438..54396562 100644 --- a/translator/app/translator/platforms/elasticsearch/renders/kibana.py +++ b/translator/app/translator/platforms/elasticsearch/renders/kibana.py @@ -16,14 +16,15 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy from typing import Optional import ujson from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.elasticsearch.const import KIBANA_RULE, KIBANA_SEARCH_SOURCE_JSON, kibana_rule_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( diff --git a/translator/app/translator/platforms/elasticsearch/renders/xpack_watcher.py b/translator/app/translator/platforms/elasticsearch/renders/xpack_watcher.py index 099a480c..e099bad7 100644 --- a/translator/app/translator/platforms/elasticsearch/renders/xpack_watcher.py +++ b/translator/app/translator/platforms/elasticsearch/renders/xpack_watcher.py @@ -16,14 +16,15 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy from typing import Optional import ujson from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.elasticsearch.const import XPACK_WATCHER_RULE, xpack_watcher_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings from app.translator.platforms.elasticsearch.renders.elasticsearch import ( diff --git a/translator/app/translator/platforms/elasticsearch/tokenizer.py b/translator/app/translator/platforms/elasticsearch/tokenizer.py index fec8fdd1..dd426e9c 100644 --- a/translator/app/translator/platforms/elasticsearch/tokenizer.py +++ b/translator/app/translator/platforms/elasticsearch/tokenizer.py @@ -15,6 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ + from app.translator.platforms.base.lucene.tokenizer import LuceneTokenizer diff --git a/translator/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/translator/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index 40a5ba2e..5eddcbaa 100644 --- a/translator/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/translator/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -24,11 +24,10 @@ from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import SourceMapping from app.translator.core.models.field import FieldValue -from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.identifier import Identifier -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender +from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.forti_siem.const import ( FORTI_SIEM_RULE, @@ -122,7 +121,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00 raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords") -class FortiSiemRuleRender(BaseQueryRender): +class FortiSiemRuleRender(PlatformQueryRender): details: PlatformDetails = forti_siem_rule_details mappings: FortiSiemMappings = forti_siem_mappings @@ -135,13 +134,13 @@ class FortiSiemRuleRender(BaseQueryRender): field_value_map = FortiSiemFieldValue(or_token=or_token) - def generate(self, query: list, meta_info: MetaInfoContainer, functions: ParsedFunctions) -> str: + def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} - source_mappings = self._get_source_mappings(meta_info.source_mapping_ids) + source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) for source_mapping in source_mappings: is_event_type_set = False - field_values = [token for token in query if isinstance(token, FieldValue)] + field_values = [token for token in query_container.tokens if isinstance(token, FieldValue)] mapped_fields_set = set() for field_value in field_values: mapped_fields = self.map_field(field_value.field, source_mapping) @@ -150,14 +149,14 @@ def generate(self, query: list, meta_info: MetaInfoContainer, functions: ParsedF is_event_type_set = True self.__update_event_type_values(field_value, source_mapping.source_id) - result = self.generate_query(query=query, source_mapping=source_mapping) + result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) prefix = "" if is_event_type_set else self.generate_prefix(source_mapping.log_source_signature) finalized_query = self.finalize_query( prefix=prefix, query=result, - functions=self.generate_functions(functions.functions, source_mapping), - not_supported_functions=functions.not_supported, - meta_info=meta_info, + functions=self.generate_functions(query_container.functions.functions, source_mapping), + not_supported_functions=query_container.functions.not_supported, + meta_info=query_container.meta_info, source_mapping=source_mapping, fields=mapped_fields_set, ) diff --git a/translator/app/translator/platforms/graylog/parsers/graylog.py b/translator/app/translator/platforms/graylog/parsers/graylog.py index 71424fd6..fe598800 100644 --- a/translator/app/translator/platforms/graylog/parsers/graylog.py +++ b/translator/app/translator/platforms/graylog/parsers/graylog.py @@ -17,11 +17,11 @@ """ from app.translator.core.models.platform_details import PlatformDetails -from app.translator.platforms.base.lucene.parsers.lucene import LuceneParser +from app.translator.platforms.base.lucene.parsers.lucene import LuceneQueryParser from app.translator.platforms.graylog.const import graylog_details from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings -class GraylogParser(LuceneParser): +class GraylogQueryParser(LuceneQueryParser): details: PlatformDetails = graylog_details mappings: GraylogMappings = graylog_mappings diff --git a/translator/app/translator/platforms/graylog/renders/graylog.py b/translator/app/translator/platforms/graylog/renders/graylog.py index 5d584b07..456a7889 100644 --- a/translator/app/translator/platforms/graylog/renders/graylog.py +++ b/translator/app/translator/platforms/graylog/renders/graylog.py @@ -27,7 +27,7 @@ class GraylogFieldValue(LuceneFieldValue): details: PlatformDetails = graylog_details -class GraylogRender(LuceneQueryRender): +class GraylogQueryRender(LuceneQueryRender): details: PlatformDetails = graylog_details mappings: GraylogMappings = graylog_mappings diff --git a/translator/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/translator/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index d182f6da..d6d34abc 100644 --- a/translator/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/translator/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -24,11 +25,10 @@ from app.translator.core.exceptions.render import BaseRenderException from app.translator.core.mapping import LogSourceSignature, SourceMapping from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.functions.base import ParsedFunctions from app.translator.core.models.identifier import Identifier -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender +from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.platforms.logrhythm_axon.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_axon_query_details from app.translator.platforms.logrhythm_axon.mapping import LogRhythmAxonMappings, logrhythm_axon_mappings from app.translator.platforms.microsoft.escape_manager import microsoft_escape_manager @@ -191,7 +191,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f'{field} matches "{value}"' -class LogRhythmAxonQueryRender(BaseQueryRender): +class LogRhythmAxonQueryRender(PlatformQueryRender): details: PlatformDetails = logrhythm_axon_query_details or_token = "OR" @@ -241,25 +241,25 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp return token.token_type - def generate(self, query: list, meta_info: MetaInfoContainer, functions: ParsedFunctions) -> str: + def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} - source_mappings = self._get_source_mappings(meta_info.source_mapping_ids) + source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) for source_mapping in source_mappings: prefix = self.generate_prefix(source_mapping.log_source_signature) - if "product" in meta_info.parsed_logsources: - prefix = f"{prefix} CONTAINS {meta_info.parsed_logsources['product'][0]}" + if "product" in query_container.meta_info.parsed_logsources: + prefix = f"{prefix} CONTAINS {query_container.meta_info.parsed_logsources['product'][0]}" else: prefix = f"{prefix} CONTAINS anything" - result = self.generate_query(query=query, source_mapping=source_mapping) + result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping) finalized_query = self.finalize_query( prefix=prefix, query=result, - functions=self.generate_functions(functions.functions, source_mapping), - not_supported_functions=functions.not_supported, - meta_info=meta_info, + functions=self.generate_functions(query_container.functions.functions, source_mapping), + not_supported_functions=query_container.functions.not_supported, + meta_info=query_container.meta_info, source_mapping=source_mapping, ) queries_map[source_mapping.source_id] = finalized_query diff --git a/translator/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py b/translator/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py index 014c9ebb..59a86013 100644 --- a/translator/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py +++ b/translator/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py @@ -16,14 +16,15 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy import json from typing import Optional from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import ( LogRhythmAxonFieldValue, diff --git a/translator/app/translator/platforms/logscale/functions/const.py b/translator/app/translator/platforms/logscale/functions/const.py index 904b3143..d9f1dee9 100644 --- a/translator/app/translator/platforms/logscale/functions/const.py +++ b/translator/app/translator/platforms/logscale/functions/const.py @@ -1,4 +1,3 @@ - from app.translator.tools.custom_enum import CustomEnum diff --git a/translator/app/translator/platforms/logscale/parsers/logscale.py b/translator/app/translator/platforms/logscale/parsers/logscale.py index 22e69242..1f2b5947 100644 --- a/translator/app/translator/platforms/logscale/parsers/logscale.py +++ b/translator/app/translator/platforms/logscale/parsers/logscale.py @@ -16,37 +16,30 @@ ----------------------------------------------------------------- """ - from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.parser import Parser +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.functions import LogScaleFunctions, log_scale_functions from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings from app.translator.platforms.logscale.tokenizer import LogScaleTokenizer -class LogScaleParser(Parser): +class LogScaleQueryParser(PlatformQueryParser): details: PlatformDetails = logscale_query_details platform_functions: LogScaleFunctions = log_scale_functions tokenizer = LogScaleTokenizer() mappings: LogScaleMappings = logscale_mappings - @staticmethod - def _get_meta_info(source_mapping_ids: list[str], metainfo: dict) -> MetaInfoContainer: # noqa: ARG004 - return MetaInfoContainer(source_mapping_ids=source_mapping_ids) - def _parse_query(self, query: str) -> tuple[str, ParsedFunctions]: - functions, query_str = self.platform_functions.parse(query) - return query_str, functions + functions, query = self.platform_functions.parse(query) + return query, functions - def parse(self, text: str) -> SiemContainer: - query, functions = self._parse_query(query=text) + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + query, functions = self._parse_query(query=raw_query_container.query) tokens, source_mappings = self.get_tokens_and_source_mappings(query, {}) self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info([source_mapping.source_id for source_mapping in source_mappings], {}), - functions=functions, - ) + meta_info = raw_query_container.meta_info + meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] + return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) diff --git a/translator/app/translator/platforms/logscale/parsers/logscale_alert.py b/translator/app/translator/platforms/logscale/parsers/logscale_alert.py index 5e225746..52eb5ec8 100644 --- a/translator/app/translator/platforms/logscale/parsers/logscale_alert.py +++ b/translator/app/translator/platforms/logscale/parsers/logscale_alert.py @@ -16,39 +16,20 @@ ----------------------------------------------------------------- """ - from app.translator.core.mixins.rule import JsonRuleMixin -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.platforms.logscale.const import logscale_alert_details -from app.translator.platforms.logscale.parsers.logscale import LogScaleParser +from app.translator.platforms.logscale.parsers.logscale import LogScaleQueryParser -class LogScaleAlertParser(LogScaleParser, JsonRuleMixin): +class LogScaleAlertParser(LogScaleQueryParser, JsonRuleMixin): details: PlatformDetails = logscale_alert_details - def _parse_rule(self, text: str) -> dict[str, str]: + def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: rule = self.load_rule(text=text) - query = rule["query"]["queryString"] - description = rule["description"] - name = rule["name"] - return {"query": query, "name": name, "description": description} - - @staticmethod - def _get_meta_info(source_mapping_ids: list[str], meta_info: dict) -> MetaInfoContainer: - return MetaInfoContainer( - title=meta_info["name"], description=meta_info["description"], source_mapping_ids=source_mapping_ids - ) - - def parse(self, text: str) -> SiemContainer: - parsed_rule = self._parse_rule(text) - query, functions = self._parse_query(query=parsed_rule.pop("query")) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, {}) - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info( - meta_info=parsed_rule, - source_mapping_ids=[source_mapping.source_id for source_mapping in source_mappings], - ), - functions=functions, + return RawQueryContainer( + query=rule["query"]["queryString"], + language=language, + meta_info=MetaInfoContainer(title=rule["name"], description=rule["description"]), ) diff --git a/translator/app/translator/platforms/logscale/renders/logscale.py b/translator/app/translator/platforms/logscale/renders/logscale.py index a7639894..9f94808a 100644 --- a/translator/app/translator/platforms/logscale/renders/logscale.py +++ b/translator/app/translator/platforms/logscale/renders/logscale.py @@ -16,13 +16,14 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender +from app.translator.core.models.query_container import MetaInfoContainer +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.escape_manager import logscale_escape_manager from app.translator.platforms.logscale.functions import LogScaleFunctions, log_scale_functions @@ -90,7 +91,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"/{self.apply_value(value)}/i" -class LogScaleQueryRender(BaseQueryRender): +class LogScaleQueryRender(PlatformQueryRender): details: PlatformDetails = logscale_query_details mappings: LogScaleMappings = logscale_mappings platform_functions: LogScaleFunctions = log_scale_functions diff --git a/translator/app/translator/platforms/logscale/renders/logscale_alert.py b/translator/app/translator/platforms/logscale/renders/logscale_alert.py index de575e8f..35050a83 100644 --- a/translator/app/translator/platforms/logscale/renders/logscale_alert.py +++ b/translator/app/translator/platforms/logscale/renders/logscale_alert.py @@ -16,13 +16,14 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy import json from typing import Optional from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValue, LogScaleQueryRender from app.translator.tools.utils import get_rule_description_str diff --git a/translator/app/translator/platforms/microsoft/parsers/microsoft_defender.py b/translator/app/translator/platforms/microsoft/parsers/microsoft_defender.py index 4468713b..576db97c 100644 --- a/translator/app/translator/platforms/microsoft/parsers/microsoft_defender.py +++ b/translator/app/translator/platforms/microsoft/parsers/microsoft_defender.py @@ -20,10 +20,10 @@ from app.translator.platforms.microsoft.const import microsoft_defender_details from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings -from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftParser +from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser -class MicrosoftDefenderQueryParser(MicrosoftParser): +class MicrosoftDefenderQueryParser(MicrosoftSentinelQueryParser): mappings: MicrosoftDefenderMappings = microsoft_defender_mappings details: PlatformDetails = microsoft_defender_details platform_functions: MicrosoftFunctions = microsoft_defender_functions diff --git a/translator/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py b/translator/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py index f07e7a45..49ba12fc 100644 --- a/translator/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/translator/app/translator/platforms/microsoft/parsers/microsoft_sentinel.py @@ -16,38 +16,31 @@ ----------------------------------------------------------------- """ - from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.parser import Parser +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_sentinel_functions from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings from app.translator.platforms.microsoft.tokenizer import MicrosoftSentinelTokenizer -class MicrosoftParser(Parser): +class MicrosoftSentinelQueryParser(PlatformQueryParser): platform_functions: MicrosoftFunctions = microsoft_sentinel_functions mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings tokenizer = MicrosoftSentinelTokenizer() details: PlatformDetails = microsoft_sentinel_query_details - @staticmethod - def _get_meta_info(source_mapping_ids: list[str], meta_info: dict) -> MetaInfoContainer: # noqa: ARG004 - return MetaInfoContainer(source_mapping_ids=source_mapping_ids) - def _parse_query(self, query: str) -> tuple[str, dict[str, list[str]], ParsedFunctions]: table, query, functions = self.platform_functions.parse(query) log_sources = {"table": [table]} return query, log_sources, functions - def parse(self, text: str) -> SiemContainer: - query, log_sources, functions = self._parse_query(query=text) + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + query, log_sources, functions = self._parse_query(query=raw_query_container.query) tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) self.set_functions_fields_generic_names(functions=functions, source_mappings=source_mappings) - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info([source_mapping.source_id for source_mapping in source_mappings], {}), - functions=functions, - ) + meta_info = raw_query_container.meta_info + meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] + return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info, functions=functions) diff --git a/translator/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py b/translator/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py index 208ac803..cde273a3 100644 --- a/translator/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py +++ b/translator/app/translator/platforms/microsoft/parsers/microsoft_sentinel_rule.py @@ -16,34 +16,20 @@ ----------------------------------------------------------------- """ - from app.translator.core.mixins.rule import JsonRuleMixin -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.platforms.microsoft.const import microsoft_sentinel_rule_details -from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftParser +from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser -class MicrosoftRuleParser(MicrosoftParser, JsonRuleMixin): +class MicrosoftSentinelRuleParser(MicrosoftSentinelQueryParser, JsonRuleMixin): details: PlatformDetails = microsoft_sentinel_rule_details - @staticmethod - def _get_meta_info(source_mapping_ids: list[str], meta_info: dict) -> MetaInfoContainer: - return MetaInfoContainer( - source_mapping_ids=source_mapping_ids, - title=meta_info.get("displayName"), - description=meta_info.get("description"), - ) - - def parse(self, text: str) -> SiemContainer: + def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: rule = self.load_rule(text=text) - query, log_sources, functions = self._parse_query(query=rule.get("query")) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info( - source_mapping_ids=[source_mapping.source_id for source_mapping in source_mappings], meta_info=rule - ), - functions=functions, + return RawQueryContainer( + query=rule["query"], + language=language, + meta_info=MetaInfoContainer(title=rule.get("displayName"), description=rule.get("description")), ) diff --git a/translator/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py b/translator/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py index 504a7fda..bdb76743 100644 --- a/translator/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py +++ b/translator/app/translator/platforms/microsoft/renders/microsoft_defender_cti.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import ClassVar from app.translator.core.models.platform_details import PlatformDetails diff --git a/translator/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/translator/app/translator/platforms/microsoft/renders/microsoft_sentinel.py index b45df389..64dd63b5 100644 --- a/translator/app/translator/platforms/microsoft/renders/microsoft_sentinel.py +++ b/translator/app/translator/platforms/microsoft/renders/microsoft_sentinel.py @@ -16,12 +16,13 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.escape_manager import microsoft_escape_manager from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_sentinel_functions @@ -101,7 +102,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"* contains @'{self.__escape_value(value)}'" -class MicrosoftSentinelQueryRender(BaseQueryRender): +class MicrosoftSentinelQueryRender(PlatformQueryRender): details: PlatformDetails = microsoft_sentinel_query_details platform_functions: MicrosoftFunctions = microsoft_sentinel_functions diff --git a/translator/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/translator/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index 9e5932d4..38c23a70 100644 --- a/translator/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/translator/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -16,14 +16,15 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy import json from typing import Optional from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( MicrosoftSentinelFieldValue, diff --git a/translator/app/translator/platforms/opensearch/parsers/opensearch.py b/translator/app/translator/platforms/opensearch/parsers/opensearch.py index 3c48ef95..76fc0752 100644 --- a/translator/app/translator/platforms/opensearch/parsers/opensearch.py +++ b/translator/app/translator/platforms/opensearch/parsers/opensearch.py @@ -17,11 +17,11 @@ """ from app.translator.core.models.platform_details import PlatformDetails -from app.translator.platforms.base.lucene.parsers.lucene import LuceneParser +from app.translator.platforms.base.lucene.parsers.lucene import LuceneQueryParser from app.translator.platforms.opensearch.const import opensearch_query_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -class OpenSearchParser(LuceneParser): +class OpenSearchQueryParser(LuceneQueryParser): details: PlatformDetails = opensearch_query_details mappings: OpenSearchMappings = opensearch_mappings diff --git a/translator/app/translator/platforms/opensearch/renders/opensearch.py b/translator/app/translator/platforms/opensearch/renders/opensearch.py index f8103456..bbb3c5af 100644 --- a/translator/app/translator/platforms/opensearch/renders/opensearch.py +++ b/translator/app/translator/platforms/opensearch/renders/opensearch.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE diff --git a/translator/app/translator/platforms/opensearch/renders/opensearch_rule.py b/translator/app/translator/platforms/opensearch/renders/opensearch_rule.py index f94c48c2..5a4e8531 100644 --- a/translator/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/translator/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + import copy from typing import Optional @@ -23,8 +24,8 @@ from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.opensearch.const import OPENSEARCH_RULE, opensearch_rule_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValue, OpenSearchQueryRender diff --git a/translator/app/translator/platforms/opensearch/tokenizer.py b/translator/app/translator/platforms/opensearch/tokenizer.py index 1cc67c6f..c07f47e2 100644 --- a/translator/app/translator/platforms/opensearch/tokenizer.py +++ b/translator/app/translator/platforms/opensearch/tokenizer.py @@ -15,6 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ----------------------------------------------------------------- """ + from app.translator.platforms.base.lucene.tokenizer import LuceneTokenizer diff --git a/translator/app/translator/platforms/qradar/parsers/qradar.py b/translator/app/translator/platforms/qradar/parsers/qradar.py index ecd1bb3f..d7e63cd3 100644 --- a/translator/app/translator/platforms/qradar/parsers/qradar.py +++ b/translator/app/translator/platforms/qradar/parsers/qradar.py @@ -19,16 +19,16 @@ import re from typing import Union -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.parser import Parser +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser from app.translator.platforms.qradar.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, qradar_query_details from app.translator.platforms.qradar.mapping import QradarMappings, qradar_mappings from app.translator.platforms.qradar.tokenizer import QradarTokenizer from app.translator.tools.utils import get_match_group -class QradarParser(Parser): +class QradarQueryParser(PlatformQueryParser): details: PlatformDetails = qradar_query_details tokenizer = QradarTokenizer() mappings: QradarMappings = qradar_mappings @@ -106,13 +106,9 @@ def _parse_query(self, text: str) -> tuple[str, dict[str, Union[list[str], list[ log_sources, query = self.__parse_log_sources(query) return query, log_sources - def _get_meta_info(self, source_mapping_ids: list[str]) -> MetaInfoContainer: - return MetaInfoContainer(source_mapping_ids=source_mapping_ids) - - def parse(self, text: str) -> SiemContainer: - query, log_sources = self._parse_query(text) + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + query, log_sources = self._parse_query(raw_query_container.query) tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info([source_mapping.source_id for source_mapping in source_mappings]), - ) + meta_info = raw_query_container.meta_info + meta_info.source_mapping_ids = [source_mapping.source_id for source_mapping in source_mappings] + return TokenizedQueryContainer(tokens=tokens, meta_info=meta_info) diff --git a/translator/app/translator/platforms/qradar/renders/qradar.py b/translator/app/translator/platforms/qradar/renders/qradar.py index 27d5635a..7685164e 100644 --- a/translator/app/translator/platforms/qradar/renders/qradar.py +++ b/translator/app/translator/platforms/qradar/renders/qradar.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Union from app.translator.const import DEFAULT_VALUE_TYPE @@ -23,7 +24,7 @@ from app.translator.core.mapping import SourceMapping from app.translator.core.models.functions.base import Function from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, BaseQueryRender +from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender from app.translator.platforms.qradar.const import qradar_query_details from app.translator.platforms.qradar.escape_manager import qradar_escape_manager from app.translator.platforms.qradar.mapping import QradarLogSourceSignature, QradarMappings, qradar_mappings @@ -108,7 +109,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"UTF8(payload) ILIKE '%{self.apply_value(value)}%'" -class QradarQueryRender(BaseQueryRender): +class QradarQueryRender(PlatformQueryRender): details: PlatformDetails = qradar_query_details mappings: QradarMappings = qradar_mappings diff --git a/translator/app/translator/platforms/roota/parsers/roota.py b/translator/app/translator/platforms/roota/parsers/roota.py index fe239c30..2beee2d0 100644 --- a/translator/app/translator/platforms/roota/parsers/roota.py +++ b/translator/app/translator/platforms/roota/parsers/roota.py @@ -20,12 +20,12 @@ from app.translator.core.exceptions.core import RootARuleValidationException, UnsupportedRootAParser from app.translator.core.mixins.rule import YamlRuleMixin -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer -from app.translator.core.parser import Parser +from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser, QueryParser from app.translator.managers import parser_manager -class RootAParser(YamlRuleMixin): +class RootAParser(QueryParser, YamlRuleMixin): parsers = parser_manager mandatory_fields: ClassVar[set[str]] = { "name", @@ -38,7 +38,7 @@ class RootAParser(YamlRuleMixin): "license", } - def __update_meta_info(self, meta_info: MetaInfoContainer, rule: dict) -> MetaInfoContainer: + def __parse_meta_info(self, rule: dict) -> MetaInfoContainer: mitre_attack = rule.get("mitre-attack") or [] mitre_tags = [i.strip("") for i in mitre_attack.split(",")] if isinstance(mitre_attack, str) else mitre_attack mitre_attack = self.parse_mitre_attack(mitre_tags) @@ -47,33 +47,38 @@ def __update_meta_info(self, meta_info: MetaInfoContainer, rule: dict) -> MetaIn rule_tags = [i.strip() for i in rule_tags.split(",")] rule_tags += mitre_tags - meta_info.title = rule.get("name") - meta_info.description = rule.get("details") - meta_info.id = rule.get("uuid", meta_info.id) - meta_info.references = rule.get("references") - meta_info.license = rule.get("license", meta_info.license) - meta_info.tags = rule_tags or meta_info.tags - meta_info.mitre_attack = mitre_attack - meta_info.date = rule.get("date", meta_info.date) - meta_info.author = rule.get("author", meta_info.author) - meta_info.severity = rule.get("severity", meta_info.severity) - return meta_info + return MetaInfoContainer( + id_=rule.get("uuid"), + title=rule.get("name"), + description=rule.get("details"), + author=rule.get("author"), + date=rule.get("date"), + license_=rule.get("license"), + severity=rule.get("severity"), + references=rule.get("references"), + mitre_attack=mitre_attack, + tags=rule_tags, + ) - def _get_parser_class(self, parser: str) -> Parser: + def __get_parser_class(self, parser: str) -> PlatformQueryParser: parser_class = self.parsers.get(parser) if parser_class: return parser_class raise UnsupportedRootAParser(parser=parser) - def __validate_rule(self, rule: dict) -> None: + def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: + rule = self.load_rule(text=text) if missing_fields := self.mandatory_fields.difference(set(rule.keys())): raise RootARuleValidationException(missing_fields=list(missing_fields)) - def parse(self, text: str) -> SiemContainer: - roota_rule = self.load_rule(text=text) - self.__validate_rule(rule=roota_rule) - detection = roota_rule.get("detection", {}).get("body", "") - parser = self._get_parser_class(roota_rule.get("detection", {}).get("language", "")) - siem_container = parser.parse(text=detection) - siem_container.meta_info = self.__update_meta_info(meta_info=siem_container.meta_info, rule=roota_rule) - return siem_container + detection = rule.get("detection", {}) + query = detection.get("body") + language = detection.get("language") + if not (query or language): + raise RootARuleValidationException(missing_fields=["detection"]) + + return RawQueryContainer(query=query, language=language, meta_info=self.__parse_meta_info(rule)) + + def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContainer: + parser = self.__get_parser_class(raw_query_container.language) + return parser.parse(raw_query_container) diff --git a/translator/app/translator/platforms/sigma/const.py b/translator/app/translator/platforms/sigma/const.py index 36811c1f..f5f4d75c 100644 --- a/translator/app/translator/platforms/sigma/const.py +++ b/translator/app/translator/platforms/sigma/const.py @@ -1,6 +1,8 @@ +SIGMA_SIEM_TYPE = "sigma" + SIGMA_RULE_DETAILS = { "name": "Sigma", - "siem_type": "sigma", + "siem_type": SIGMA_SIEM_TYPE, "platform_name": "Sigma", "group_name": "Sigma", "group_id": "sigma", diff --git a/translator/app/translator/platforms/sigma/parsers/sigma.py b/translator/app/translator/platforms/sigma/parsers/sigma.py index f2139e7c..c8a9b2a8 100644 --- a/translator/app/translator/platforms/sigma/parsers/sigma.py +++ b/translator/app/translator/platforms/sigma/parsers/sigma.py @@ -17,12 +17,13 @@ ----------------------------------------------------------------- """ + from typing import Union from app.translator.core.exceptions.core import SigmaRuleValidationException from app.translator.core.mixins.rule import YamlRuleMixin from app.translator.core.models.field import FieldValue, Field -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer +from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.tokenizer import QueryTokenizer from app.translator.platforms.sigma.const import SIGMA_RULE_DETAILS @@ -72,7 +73,7 @@ def __validate_rule(self, rule: dict): if missing_fields := self.mandatory_fields.difference(set(rule.keys())): raise SigmaRuleValidationException(missing_fields=list(missing_fields)) - def parse(self, text: str) -> SiemContainer: + def parse(self, text: str) -> TokenizedQueryContainer: sigma_rule = self.load_rule(text=text) self.__validate_rule(rule=sigma_rule) log_sources = { @@ -90,8 +91,8 @@ def parse(self, text: str) -> SiemContainer: sigma_fields_tokens = [Field(source_name=field) for field in sigma_fields] QueryTokenizer.set_field_tokens_generic_names_map(sigma_fields_tokens, source_mappings, self.mappings.default_mapping) - return SiemContainer( - query=tokens, + return TokenizedQueryContainer( + tokens=tokens, meta_info=self._get_meta_info( rule=sigma_rule, source_mapping_ids=[source_mapping.source_id for source_mapping in source_mappings], diff --git a/translator/app/translator/platforms/sigma/renders/sigma.py b/translator/app/translator/platforms/sigma/renders/sigma.py index d3ecb183..6594fdb4 100644 --- a/translator/app/translator/platforms/sigma/renders/sigma.py +++ b/translator/app/translator/platforms/sigma/renders/sigma.py @@ -16,7 +16,6 @@ ----------------------------------------------------------------- """ -import copy from typing import Any, Union import yaml @@ -26,9 +25,9 @@ from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.mapping import DEFAULT_MAPPING_NAME, SourceMapping from app.translator.core.models.field import FieldValue, Keyword -from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.models.parser_output import MetaInfoContainer +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.render import QueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.sigma.const import SIGMA_RULE_DETAILS from app.translator.platforms.sigma.mapping import SigmaLogSourceSignature, SigmaMappings, sigma_mappings @@ -38,7 +37,7 @@ from app.translator.platforms.sigma.str_value_manager import sigma_str_value_manager -class SigmaRender: +class SigmaRender(QueryRender): selection_name = "selection" selection_num = 0 keyword_name = "keyword" @@ -275,13 +274,16 @@ def __get_source_mapping(self, source_mapping_ids: list[str]) -> SourceMapping: return self.mappings.get_source_mapping(DEFAULT_MAPPING_NAME) - def generate(self, query, meta_info: MetaInfoContainer, functions: ParsedFunctions): + def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + raise NotImplementedError + + def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: self.reset_counters() + meta_info = query_container.meta_info source_mapping = self.__get_source_mapping(meta_info.source_mapping_ids) log_source_signature: SigmaLogSourceSignature = source_mapping.log_source_signature - sigma_condition = copy.deepcopy(query) - prepared_data_structure = DataStructureCompiler().generate(tokens=sigma_condition) + prepared_data_structure = DataStructureCompiler().generate(tokens=query_container.tokens) rule = { "title": meta_info.title or "Autogenerated Sigma Rule", @@ -298,3 +300,9 @@ def generate(self, query, meta_info: MetaInfoContainer, functions: ParsedFunctio "falsepositives": "", } return yaml.dump(rule, default_flow_style=False, sort_keys=False) + + def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: + if isinstance(query_container, RawQueryContainer): + return self._generate_from_raw_query_container(query_container) + + return self._generate_from_tokenized_query_container(query_container) diff --git a/translator/app/translator/platforms/splunk/parsers/splunk.py b/translator/app/translator/platforms/splunk/parsers/splunk.py index a3fa2ad2..02300eaf 100644 --- a/translator/app/translator/platforms/splunk/parsers/splunk.py +++ b/translator/app/translator/platforms/splunk/parsers/splunk.py @@ -17,13 +17,13 @@ """ from app.translator.core.models.platform_details import PlatformDetails -from app.translator.platforms.base.spl.parsers.spl import SplParser +from app.translator.platforms.base.spl.parsers.spl import SplQueryParser from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings -class SplunkParser(SplParser): +class SplunkQueryParser(SplQueryParser): details: PlatformDetails = splunk_query_details log_source_pattern = r"^___source_type___\s*=\s*(?:\"(?P[%a-zA-Z_*:0-9\-/]+)\"|(?P[%a-zA-Z_*:0-9\-/]+))(?:\s+(?:and|or)\s+|\s+)?" # noqa: E501 diff --git a/translator/app/translator/platforms/splunk/parsers/splunk_alert.py b/translator/app/translator/platforms/splunk/parsers/splunk_alert.py index ee8476a1..9d874a3c 100644 --- a/translator/app/translator/platforms/splunk/parsers/splunk_alert.py +++ b/translator/app/translator/platforms/splunk/parsers/splunk_alert.py @@ -17,30 +17,17 @@ """ import re -from typing import Optional -from app.translator.core.models.parser_output import MetaInfoContainer, SiemContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer from app.translator.platforms.splunk.const import splunk_alert_details -from app.translator.platforms.splunk.parsers.splunk import SplunkParser +from app.translator.platforms.splunk.parsers.splunk import SplunkQueryParser -class SplunkAlertParser(SplunkParser): +class SplunkAlertParser(SplunkQueryParser): details: PlatformDetails = splunk_alert_details - @staticmethod - def _get_meta_info(source_mapping_ids: list[str], meta_info: Optional[str]) -> MetaInfoContainer: - description = re.search(r"description\s*=\s*(?P.+)", meta_info).group() - description = description.replace("description = ", "") - return MetaInfoContainer(source_mapping_ids=source_mapping_ids, description=description) - - def parse(self, text: str) -> SiemContainer: + def parse_raw_query(self, text: str, language: str) -> RawQueryContainer: query = re.search(r"search\s*=\s*(?P.+)", text).group("query") - log_sources, functions, query = self._parse_query(query) - tokens, source_mappings = self.get_tokens_and_source_mappings(query, log_sources) - - return SiemContainer( - query=tokens, - meta_info=self._get_meta_info([source_mapping.source_id for source_mapping in source_mappings], text), - functions=functions, - ) + description = re.search(r"description\s*=\s*(?P.+)", text).group("description") + return RawQueryContainer(query=query, language=language, meta_info=MetaInfoContainer(description=description)) diff --git a/translator/app/translator/platforms/splunk/renders/splunk.py b/translator/app/translator/platforms/splunk/renders/splunk.py index 7e5e59c5..3a1c10ca 100644 --- a/translator/app/translator/platforms/splunk/renders/splunk.py +++ b/translator/app/translator/platforms/splunk/renders/splunk.py @@ -16,6 +16,7 @@ limitations under the License. ----------------------------------------------------------------- """ + from app.translator.core.models.platform_details import PlatformDetails from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender from app.translator.platforms.splunk.const import splunk_query_details diff --git a/translator/app/translator/platforms/splunk/renders/splunk_alert.py b/translator/app/translator/platforms/splunk/renders/splunk_alert.py index c229feb7..2218ca42 100644 --- a/translator/app/translator/platforms/splunk/renders/splunk_alert.py +++ b/translator/app/translator/platforms/splunk/renders/splunk_alert.py @@ -16,12 +16,13 @@ limitations under the License. ----------------------------------------------------------------- """ + from typing import Optional from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import SourceMapping -from app.translator.core.models.parser_output import MetaInfoContainer from app.translator.core.models.platform_details import PlatformDetails +from app.translator.core.models.query_container import MetaInfoContainer from app.translator.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details from app.translator.platforms.splunk.renders.splunk import SplunkFieldValue, SplunkQueryRender from app.translator.tools.utils import get_rule_description_str diff --git a/translator/app/translator/tools/decorators.py b/translator/app/translator/tools/decorators.py index 32e6f3f9..59e8f94c 100644 --- a/translator/app/translator/tools/decorators.py +++ b/translator/app/translator/tools/decorators.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from typing import Any from app.translator.core.exceptions.core import BasePlatformException from app.translator.core.exceptions.iocs import BaseIOCsException @@ -6,8 +7,8 @@ from app.translator.core.exceptions.render import BaseRenderException -def handle_translation_exceptions(func: Callable[..., ...]) -> Callable[..., tuple[bool, str]]: - def exception_handler(*args, **kwargs) -> tuple[bool, str]: +def handle_translation_exceptions(func: Callable[..., ...]) -> Callable[..., tuple[bool, Any]]: + def exception_handler(*args, **kwargs) -> tuple[bool, Any]: try: result = func(*args, **kwargs) except (BaseParserException, BasePlatformException, BaseRenderException, BaseIOCsException) as err: diff --git a/translator/app/translator/translator.py b/translator/app/translator/translator.py index 75a48b31..8791d4c7 100644 --- a/translator/app/translator/translator.py +++ b/translator/app/translator/translator.py @@ -1,9 +1,14 @@ import logging +from typing import Optional, Union from app.translator.core.exceptions.core import UnsupportedPlatform -from app.translator.core.models.parser_output import SiemContainer +from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer +from app.translator.core.parser import PlatformQueryParser +from app.translator.core.render import QueryRender from app.translator.managers import ParserManager, RenderManager, parser_manager, render_manager +from app.translator.platforms.elasticsearch.const import ELASTIC_QUERY_TYPES from app.translator.platforms.roota.parsers.roota import RootAParser +from app.translator.platforms.sigma.parsers.sigma import SigmaParser from app.translator.tools.decorators import handle_translation_exceptions @@ -14,44 +19,80 @@ class SiemConverter: def __init__(self): self.logger = logging.getLogger("siem_converter") - @handle_translation_exceptions - def __parse_incoming_data(self, text: str, source: str) -> SiemContainer: + def __get_parser(self, source: str) -> Union[PlatformQueryParser, RootAParser, SigmaParser]: parser = RootAParser() if source == "roota" else self.parsers.get(source) if not parser: raise UnsupportedPlatform(platform=source, is_parser=True) - return parser.parse(text=text) - @handle_translation_exceptions - def __render_translation(self, parsed_data: SiemContainer, target: str) -> str: - render = self.renders.get(target) - if not render: + return parser + + def __get_render(self, target: str) -> QueryRender: + if not (render := self.renders.get(target)): raise UnsupportedPlatform(platform=target) - return render.generate( - query=parsed_data.query, meta_info=parsed_data.meta_info, functions=parsed_data.functions - ) - def __generate_one_translation(self, text: str, source: str, target: str) -> (bool, str): - status, parsed_data = self.__parse_incoming_data(text=text, source=source) - if status: - return self.__render_translation(parsed_data=parsed_data, target=target) - return status, parsed_data + return render + + @staticmethod + def __is_one_vendor_translation(source: str, target: str) -> bool: + vendors_query_types = [ELASTIC_QUERY_TYPES] + for vendor_query_types in vendors_query_types: + if source in vendor_query_types and target in vendor_query_types: + return True + + return False + + @handle_translation_exceptions + def __parse_incoming_data( + self, text: str, source: str, target: Optional[str] = None + ) -> tuple[Optional[RawQueryContainer], Optional[TokenizedQueryContainer]]: + parser = self.__get_parser(source) + if isinstance(parser, SigmaParser): + return None, parser.parse(text) + + raw_query_container = parser.parse_raw_query(text, language=source) + tokenized_query_container = None + if target and not self.__is_one_vendor_translation(raw_query_container.language, target): + tokenized_query_container = parser.parse(raw_query_container) + + return raw_query_container, tokenized_query_container + + @handle_translation_exceptions + def __render_translation( + self, query_container: Union[RawQueryContainer, TokenizedQueryContainer], target: str + ) -> str: + render = self.__get_render(target) + return render.generate(query_container) + + def __generate_one(self, text: str, source: str, target: str) -> (bool, str): + status, parsed_data = self.__parse_incoming_data(text=text, source=source, target=target) + if not status: + return status, parsed_data + + raw_query_container, tokenized_query_container = parsed_data + query_container = tokenized_query_container or raw_query_container + return self.__render_translation(query_container=query_container, target=target) def __generate_all(self, text: str, source: str) -> list[dict]: status, parsed_data = self.__parse_incoming_data(text=text, source=source) - if status: - result = [] - for target in self.renders.all_platforms(): - if target == source: - continue - translation = {"siem_type": target} - render_status, data = self.__render_translation(parsed_data=parsed_data, target=target) - translation.update({"status": render_status, "result": data}) - result.append(translation) - return result - return [{"status": status, "result": parsed_data, "siem_type": source}] + if not status: + return [{"status": status, "result": parsed_data, "siem_type": source}] + + raw_query_container, tokenized_query_container = parsed_data + result = [] + for target in self.renders.all_platforms(): + if target == source: + continue + + if raw_query_container and self.__is_one_vendor_translation(raw_query_container.language, target): + status, data = self.__render_translation(query_container=raw_query_container, target=target) + else: + status, data = self.__render_translation(query_container=tokenized_query_container, target=target) + result.append({"status": status, "result": parsed_data, "siem_type": target}) + + return result def generate_translation(self, text: str, source: str, target: str) -> (bool, str): - return self.__generate_one_translation(text=text, source=source, target=target) + return self.__generate_one(text=text, source=source, target=target) def generate_all_translation(self, text: str, source: str) -> list[dict]: return self.__generate_all(text=text, source=source) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy