UncoderIO · alexvolha · Nov 28, 2023 · Nov 28, 2023
diff --git a/siem-converter/app/converter/converter.py b/siem-converter/app/converter/converter.py
@@ -2,7 +2,7 @@
 
 from app.converter.platforms.roota.parsers.roota import RootAParser
 from app.converter.core.exceptions.core import UnsupportedPlatform
-from app.converter.core.operator_types.output import SiemContainer
+from app.converter.core.models.parser_output import SiemContainer
 from app.converter.managers import RenderManager, ParserManager, render_manager, parser_manager
 from app.converter.tools.decorators import handle_translation_exceptions
 

diff --git a/...converter/core/operator_types/__init__.py → ...p/converter/core/custom_types/__init__.py b/...converter/core/operator_types/__init__.py → ...p/converter/core/custom_types/__init__.py
diff --git a/...p/converter/core/operator_types/tokens.py → ...app/converter/core/custom_types/tokens.py b/...p/converter/core/operator_types/tokens.py → ...app/converter/core/custom_types/tokens.py
@@ -14,7 +14,6 @@ class OperatorType(CustomEnum):
     GTE = ">="
     EQ = "="
     NEQ = "!="
-    COLON = ":"
     CONTAINS = "contains"
     STARTSWITH = "startswith"
     ENDSWITH = "endswith"
@@ -25,8 +24,3 @@ class OperatorType(CustomEnum):
 class GroupType(CustomEnum):
     L_PAREN = "("
     R_PAREN = ")"
-    GROUP = "group"
-
-
-class ValidTokens(LogicalOperatorType, OperatorType, GroupType):
-    pass
diff --git a/siem-converter/app/converter/core/mixins/operator.py b/siem-converter/app/converter/core/mixins/operator.py
@@ -19,7 +19,7 @@
 from typing import Union, List, Tuple
 
 from app.converter.core.models.identifier import Identifier
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.core.custom_types.tokens import OperatorType
 
 
 class WildCardMixin:

diff --git a/siem-converter/app/converter/core/models/field.py b/siem-converter/app/converter/core/models/field.py
@@ -2,7 +2,7 @@
 
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.identifier import Identifier
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.core.custom_types.tokens import OperatorType
 
 
 class Field:

diff --git a/siem-converter/app/converter/core/models/functions/table.py b/siem-converter/app/converter/core/models/functions/table.py
@@ -1,4 +1,4 @@
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.core.custom_types.tokens import OperatorType
 
 
 class TableField:

diff --git a/siem-converter/app/converter/core/models/identifier.py b/siem-converter/app/converter/core/models/identifier.py
@@ -1,12 +1,16 @@
 from dataclasses import dataclass
 
-from app.converter.core.operator_types.tokens import ValidTokens
+from app.converter.core.custom_types.tokens import LogicalOperatorType, OperatorType, GroupType
+
+
+class _IdentifierTokenType(LogicalOperatorType, OperatorType, GroupType):
+    pass
 
 
 @dataclass
 class Identifier:
     def __init__(self, *, token_type: str) -> None:
-        if token_type not in ValidTokens:
+        if token_type not in _IdentifierTokenType:
             raise Exception(f"Unexpected token type: {token_type}")
 
         self.token_type = token_type

diff --git a/...p/converter/core/operator_types/output.py → ...pp/converter/core/models/parser_output.py b/...p/converter/core/operator_types/output.py → ...pp/converter/core/models/parser_output.py
diff --git a/siem-converter/app/converter/core/parser.py b/siem-converter/app/converter/core/parser.py
@@ -22,7 +22,7 @@
 from app.converter.core.mapping import BasePlatformMappings, SourceMapping
 from app.converter.core.models.field import Field
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 from app.converter.core.tokenizer import QueryTokenizer, TOKEN_TYPE
 
 

diff --git a/siem-converter/app/converter/core/render.py b/siem-converter/app/converter/core/render.py
@@ -26,8 +26,8 @@
 from app.converter.core.models.field import Field, Keyword
 from app.converter.core.models.functions.types import ParsedFunctions
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
-from app.converter.core.operator_types.tokens import LogicalOperatorType, OperatorType, GroupType
+from app.converter.core.models.parser_output import MetaInfoContainer
+from app.converter.core.custom_types.tokens import LogicalOperatorType, OperatorType, GroupType
 
 
 class BaseQueryFieldValue(ABC):

diff --git a/siem-converter/app/converter/core/tokenizer.py b/siem-converter/app/converter/core/tokenizer.py
@@ -27,8 +27,8 @@
 from app.converter.core.mapping import SourceMapping, DEFAULT_MAPPING_NAME, BasePlatformMappings
 from app.converter.core.models.field import Field, Keyword
 from app.converter.core.models.identifier import Identifier
-from app.converter.core.models.group import GroupType
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.platforms.sigma.models.group import GroupType
+from app.converter.core.custom_types.tokens import OperatorType
 from app.converter.tools.utils import get_match_group
 
 TOKEN_TYPE = Union[Field, Keyword, Identifier]

diff --git a/siem-converter/app/converter/platforms/athena/parsers/athena.py b/siem-converter/app/converter/platforms/athena/parsers/athena.py
@@ -24,7 +24,7 @@
 from app.converter.platforms.athena.tokenizer import AthenaTokenizer
 from app.converter.core.models.platform_details import PlatformDetails
 from app.converter.core.parser import Parser
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class AthenaParser(Parser):

diff --git a/siem-converter/app/converter/platforms/athena/tokenizer.py b/siem-converter/app/converter/platforms/athena/tokenizer.py
@@ -21,7 +21,7 @@
 
 from app.converter.core.models.identifier import Identifier
 from app.converter.core.tokenizer import QueryTokenizer
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.core.custom_types.tokens import OperatorType
 from app.converter.tools.utils import get_match_group
 
 

diff --git a/siem-converter/app/converter/platforms/base/lucene/parsers/lucene.py b/siem-converter/app/converter/platforms/base/lucene/parsers/lucene.py
@@ -21,7 +21,7 @@
 
 from app.converter.platforms.base.lucene.tokenizer import LuceneTokenizer
 from app.converter.core.parser import Parser
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class LuceneParser(Parser):

diff --git a/siem-converter/app/converter/platforms/base/lucene/tokenizer.py b/siem-converter/app/converter/platforms/base/lucene/tokenizer.py
@@ -23,7 +23,7 @@
 from app.converter.core.models.field import Keyword, Field
 from app.converter.core.models.identifier import Identifier
 from app.converter.core.tokenizer import QueryTokenizer
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.core.custom_types.tokens import OperatorType
 from app.converter.tools.utils import get_match_group
 
 

diff --git a/siem-converter/app/converter/platforms/base/spl/parsers/spl.py b/siem-converter/app/converter/platforms/base/spl/parsers/spl.py
@@ -22,7 +22,7 @@
 from app.converter.platforms.base.spl.tokenizer import SplTokenizer
 from app.converter.core.models.functions.types import ParsedFunctions
 from app.converter.core.parser import Parser
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class SplParser(Parser):

diff --git a/siem-converter/app/converter/platforms/base/spl/tokenizer.py b/siem-converter/app/converter/platforms/base/spl/tokenizer.py
@@ -20,7 +20,7 @@
 from typing import Tuple, Any
 
 from app.converter.core.tokenizer import QueryTokenizer
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.core.custom_types.tokens import OperatorType
 from app.converter.tools.utils import get_match_group
 
 

diff --git a/siem-converter/app/converter/platforms/chronicle/parsers/chronicle.py b/siem-converter/app/converter/platforms/chronicle/parsers/chronicle.py
@@ -23,7 +23,7 @@
 from app.converter.platforms.chronicle.tokenizer import ChronicleQueryTokenizer
 from app.converter.core.models.platform_details import PlatformDetails
 from app.converter.core.parser import Parser
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class ChronicleParser(Parser):

diff --git a/siem-converter/app/converter/platforms/chronicle/parsers/chronicle_rule.py b/siem-converter/app/converter/platforms/chronicle/parsers/chronicle_rule.py
@@ -25,7 +25,7 @@
 from app.converter.core.exceptions.parser import TokenizerGeneralException
 from app.converter.core.models.platform_details import PlatformDetails
 from app.converter.core.parser import Parser
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class ChronicleRuleParser(Parser):

diff --git a/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py b/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py
@@ -23,7 +23,7 @@
 from app.converter.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
+from app.converter.core.models.parser_output import MetaInfoContainer
 from app.converter.tools.utils import concatenate_str, get_author_str
 
 _AUTOGENERATED_TITLE = "Autogenerated Chronicle Security rule"

diff --git a/siem-converter/app/converter/platforms/chronicle/tokenizer.py b/siem-converter/app/converter/platforms/chronicle/tokenizer.py
@@ -21,7 +21,7 @@
 
 from app.converter.core.exceptions.parser import TokenizerGeneralException
 from app.converter.core.tokenizer import QueryTokenizer
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.core.custom_types.tokens import OperatorType
 from app.converter.tools.utils import get_match_group
 
 

diff --git a/siem-converter/app/converter/platforms/elasticsearch/parsers/detection_rule.py b/siem-converter/app/converter/platforms/elasticsearch/parsers/detection_rule.py
@@ -22,7 +22,7 @@
 from app.converter.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchParser
 from app.converter.core.mixins.rule import JsonRuleMixin
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class ElasticSearchRuleParser(ElasticSearchParser, JsonRuleMixin):

diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py b/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py
@@ -25,7 +25,7 @@
 from app.converter.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender, ElasticSearchFieldValue
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
+from app.converter.core.models.parser_output import MetaInfoContainer
 from app.converter.tools.utils import concatenate_str, get_mitre_attack_str
 
 

diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py b/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py
@@ -22,7 +22,7 @@
 from app.converter.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender, ElasticSearchFieldValue
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
+from app.converter.core.models.parser_output import MetaInfoContainer
 from app.converter.tools.utils import get_author_str, concatenate_str, get_mitre_attack_str, get_licence_str
 
 

diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py b/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py
@@ -25,7 +25,7 @@
 from app.converter.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender, ElasticSearchFieldValue
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
+from app.converter.core.models.parser_output import MetaInfoContainer
 from app.converter.tools.utils import concatenate_str, get_author_str, get_licence_str, get_mitre_attack_str, \
     get_rule_id_str, get_references_str
 

diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py b/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py
@@ -25,7 +25,7 @@
 from app.converter.platforms.elasticsearch.const import XPACK_WATCHER_RULE, xpack_watcher_details
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
+from app.converter.core.models.parser_output import MetaInfoContainer
 from app.converter.tools.utils import concatenate_str, get_author_str, get_licence_str, get_mitre_attack_str
 
 

diff --git a/siem-converter/app/converter/platforms/logscale/parsers/logscale.py b/siem-converter/app/converter/platforms/logscale/parsers/logscale.py
@@ -25,7 +25,7 @@
 from app.converter.platforms.logscale.tokenizer import LogScaleTokenizer
 from app.converter.core.models.platform_details import PlatformDetails
 from app.converter.core.parser import Parser
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class LogScaleParser(Parser):

diff --git a/siem-converter/app/converter/platforms/logscale/parsers/logscale_alert.py b/siem-converter/app/converter/platforms/logscale/parsers/logscale_alert.py
@@ -22,7 +22,7 @@
 from app.converter.platforms.logscale.parsers.logscale import LogScaleParser
 from app.converter.core.mixins.rule import JsonRuleMixin
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class LogScaleAlertParser(LogScaleParser, JsonRuleMixin):

diff --git a/siem-converter/app/converter/platforms/logscale/renders/logscale.py b/siem-converter/app/converter/platforms/logscale/renders/logscale.py
@@ -22,7 +22,7 @@
 from app.converter.platforms.logscale.mapping import LogScaleMappings, logscale_mappings
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
+from app.converter.core.models.parser_output import MetaInfoContainer
 from app.converter.core.render import BaseQueryRender, BaseQueryFieldValue
 
 

diff --git a/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py b/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py
@@ -24,7 +24,7 @@
 from app.converter.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
+from app.converter.core.models.parser_output import MetaInfoContainer
 
 
 _AUTOGENERATED_TITLE = "Autogenerated Falcon LogScale Alert"

diff --git a/siem-converter/app/converter/platforms/logscale/tokenizer.py b/siem-converter/app/converter/platforms/logscale/tokenizer.py
@@ -21,7 +21,7 @@
 
 from app.converter.core.models.field import Keyword, Field
 from app.converter.core.models.identifier import Identifier
-from app.converter.core.operator_types.tokens import GroupType, LogicalOperatorType, OperatorType
+from app.converter.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType
 from app.converter.core.tokenizer import QueryTokenizer
 from app.converter.tools.utils import get_match_group
 

diff --git a/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel.py b/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel.py
@@ -25,7 +25,7 @@
 from app.converter.core.models.functions.types import ParsedFunctions
 from app.converter.core.models.platform_details import PlatformDetails
 from app.converter.core.parser import Parser
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class MicrosoftParser(Parser):

diff --git a/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel_rule.py b/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel_rule.py
@@ -22,7 +22,7 @@
 from app.converter.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftParser
 from app.converter.core.mixins.rule import JsonRuleMixin
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 
 
 class MicrosoftRuleParser(MicrosoftParser, JsonRuleMixin):

diff --git a/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py b/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py
@@ -19,7 +19,6 @@
 
 import copy
 import json
-import re
 
 from app.converter.platforms.microsoft.renders.microsoft_sentinel import (
     MicrosoftSentinelQueryRender,
@@ -28,7 +27,7 @@
 from app.converter.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
+from app.converter.core.models.parser_output import MetaInfoContainer
 from app.converter.tools.utils import concatenate_str, get_author_str, get_licence_str
 
 

diff --git a/siem-converter/app/converter/platforms/microsoft/tokenizer.py b/siem-converter/app/converter/platforms/microsoft/tokenizer.py
@@ -21,7 +21,7 @@
 
 from app.converter.core.mixins.operator import OperatorBasedMixin
 from app.converter.core.tokenizer import QueryTokenizer
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.core.custom_types.tokens import OperatorType
 from app.converter.tools.utils import get_match_group
 
 

diff --git a/siem-converter/app/converter/platforms/opensearch/renders/opensearch_rule.py b/siem-converter/app/converter/platforms/opensearch/renders/opensearch_rule.py
@@ -25,7 +25,7 @@
 from app.converter.platforms.opensearch.renders.opensearch import OpenSearchQueryRender, OpenSearchFieldValue
 from app.converter.core.mapping import SourceMapping
 from app.converter.core.models.platform_details import PlatformDetails
-from app.converter.core.operator_types.output import MetaInfoContainer
+from app.converter.core.models.parser_output import MetaInfoContainer
 
 
 SEVERITIES_MAP = {"informational": "5", "low": "4", "medium": "3", "high": "2", "critical": "1"}

diff --git a/siem-converter/app/converter/platforms/qradar/parsers/qradar.py b/siem-converter/app/converter/platforms/qradar/parsers/qradar.py
@@ -25,7 +25,7 @@
 from app.converter.platforms.qradar.tokenizer import QradarTokenizer
 from app.converter.core.models.platform_details import PlatformDetails
 from app.converter.core.parser import Parser
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 from app.converter.tools.utils import get_match_group
 
 

diff --git a/siem-converter/app/converter/platforms/qradar/tokenizer.py b/siem-converter/app/converter/platforms/qradar/tokenizer.py
@@ -23,7 +23,7 @@
 from app.converter.core.models.field import Keyword
 from app.converter.core.models.identifier import Identifier
 from app.converter.core.tokenizer import QueryTokenizer
-from app.converter.core.operator_types.tokens import OperatorType
+from app.converter.core.custom_types.tokens import OperatorType
 from app.converter.tools.utils import get_match_group
 
 

diff --git a/siem-converter/app/converter/platforms/roota/parsers/roota.py b/siem-converter/app/converter/platforms/roota/parsers/roota.py
@@ -18,7 +18,7 @@
 
 from app.converter.core.exceptions.core import UnsupportedRootAParser, RootARuleValidationException
 from app.converter.core.mixins.rule import YamlRuleMixin
-from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer
+from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer
 from app.converter.core.parser import Parser
 from app.converter.managers import parser_manager
 

diff --git a/...-converter/app/converter/core/compiler.py → ...verter/platforms/sigma/models/compiler.py b/...-converter/app/converter/core/compiler.py → ...verter/platforms/sigma/models/compiler.py
@@ -17,10 +17,10 @@
 """
 
 from app.converter.core.models.field import Field, Keyword
-from app.converter.core.models.group import Group
+from app.converter.platforms.sigma.models.group import Group
 from app.converter.core.models.identifier import Identifier
-from app.converter.core.models.operator import Operator, NOT
-from app.converter.core.operator_types.tokens import LogicalOperatorType, GroupType
+from app.converter.platforms.sigma.models.operator import Operator, NOT
+from app.converter.core.custom_types.tokens import LogicalOperatorType, GroupType
 
 
 class DataStructureCompiler:

diff --git a/...verter/app/converter/core/models/group.py → ...converter/platforms/sigma/models/group.py b/...verter/app/converter/core/models/group.py → ...converter/platforms/sigma/models/group.py
@@ -1,9 +1,7 @@
-from app.converter.core.models.operator import OR, AND, NOT
-from app.converter.core.operator_types.tokens import GroupType
+from app.converter.platforms.sigma.models.operator import OR, AND, NOT
 
 
 class Group:
-    token_type = GroupType.GROUP
     parent_group = []
     sub_group = None
     last_field = None

diff --git a/siem-converter/app/converter/platforms/sigma/models/modifiers.py b/siem-converter/app/converter/platforms/sigma/models/modifiers.py
@@ -2,7 +2,7 @@
 
 from app.converter.core.models.field import Field
 from app.converter.core.models.identifier import Identifier
-from app.converter.core.operator_types.tokens import LogicalOperatorType, OperatorType, GroupType
+from app.converter.core.custom_types.tokens import LogicalOperatorType, OperatorType, GroupType
 
 
 class ModifierManager: