From 19f4372f4dd583bda48ed374b92afc09d5ea2649 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Tue, 28 Nov 2023 09:45:30 +0200 Subject: [PATCH 001/542] files structure refactoring --- siem-converter/app/converter/converter.py | 2 +- .../core/{operator_types => custom_types}/__init__.py | 0 .../core/{operator_types => custom_types}/tokens.py | 6 ------ siem-converter/app/converter/core/mixins/operator.py | 2 +- siem-converter/app/converter/core/models/field.py | 2 +- .../app/converter/core/models/functions/table.py | 2 +- siem-converter/app/converter/core/models/identifier.py | 8 ++++++-- .../output.py => models/parser_output.py} | 0 siem-converter/app/converter/core/parser.py | 2 +- siem-converter/app/converter/core/render.py | 4 ++-- siem-converter/app/converter/core/tokenizer.py | 4 ++-- .../app/converter/platforms/athena/parsers/athena.py | 2 +- .../app/converter/platforms/athena/tokenizer.py | 2 +- .../converter/platforms/base/lucene/parsers/lucene.py | 2 +- .../app/converter/platforms/base/lucene/tokenizer.py | 2 +- .../app/converter/platforms/base/spl/parsers/spl.py | 2 +- .../app/converter/platforms/base/spl/tokenizer.py | 2 +- .../converter/platforms/chronicle/parsers/chronicle.py | 2 +- .../platforms/chronicle/parsers/chronicle_rule.py | 2 +- .../platforms/chronicle/renders/chronicle_rule.py | 2 +- .../app/converter/platforms/chronicle/tokenizer.py | 2 +- .../platforms/elasticsearch/parsers/detection_rule.py | 2 +- .../platforms/elasticsearch/renders/detection_rule.py | 2 +- .../platforms/elasticsearch/renders/elast_alert.py | 2 +- .../platforms/elasticsearch/renders/kibana.py | 2 +- .../platforms/elasticsearch/renders/xpack_watcher.py | 2 +- .../converter/platforms/logscale/parsers/logscale.py | 2 +- .../platforms/logscale/parsers/logscale_alert.py | 2 +- .../converter/platforms/logscale/renders/logscale.py | 2 +- .../platforms/logscale/renders/logscale_alert.py | 2 +- .../app/converter/platforms/logscale/tokenizer.py | 2 +- .../platforms/microsoft/parsers/microsoft_sentinel.py | 2 +- .../microsoft/parsers/microsoft_sentinel_rule.py | 2 +- .../microsoft/renders/microsoft_sentinel_rule.py | 3 +-- .../app/converter/platforms/microsoft/tokenizer.py | 2 +- .../platforms/opensearch/renders/opensearch_rule.py | 2 +- .../app/converter/platforms/qradar/parsers/qradar.py | 2 +- .../app/converter/platforms/qradar/tokenizer.py | 2 +- .../app/converter/platforms/roota/parsers/roota.py | 2 +- .../{core => platforms/sigma/models}/compiler.py | 6 +++--- .../{core => platforms/sigma}/models/group.py | 4 +--- .../app/converter/platforms/sigma/models/modifiers.py | 2 +- .../{core => platforms/sigma}/models/operator.py | 2 +- .../app/converter/platforms/sigma/parsers/sigma.py | 2 +- .../app/converter/platforms/sigma/renders/sigma.py | 10 +++++----- .../app/converter/platforms/sigma/tokenizer.py | 2 +- .../converter/platforms/splunk/parsers/splunk_alert.py | 2 +- .../converter/platforms/splunk/renders/splunk_alert.py | 2 +- 48 files changed, 58 insertions(+), 63 deletions(-) rename siem-converter/app/converter/core/{operator_types => custom_types}/__init__.py (100%) rename siem-converter/app/converter/core/{operator_types => custom_types}/tokens.py (80%) rename siem-converter/app/converter/core/{operator_types/output.py => models/parser_output.py} (100%) rename siem-converter/app/converter/{core => platforms/sigma/models}/compiler.py (92%) rename siem-converter/app/converter/{core => platforms/sigma}/models/group.py (93%) rename siem-converter/app/converter/{core => platforms/sigma}/models/operator.py (93%) diff --git a/siem-converter/app/converter/converter.py b/siem-converter/app/converter/converter.py index 87652d46..5d1baefb 100644 --- a/siem-converter/app/converter/converter.py +++ b/siem-converter/app/converter/converter.py @@ -2,7 +2,7 @@ from app.converter.platforms.roota.parsers.roota import RootAParser from app.converter.core.exceptions.core import UnsupportedPlatform -from app.converter.core.operator_types.output import SiemContainer +from app.converter.core.models.parser_output import SiemContainer from app.converter.managers import RenderManager, ParserManager, render_manager, parser_manager from app.converter.tools.decorators import handle_translation_exceptions diff --git a/siem-converter/app/converter/core/operator_types/__init__.py b/siem-converter/app/converter/core/custom_types/__init__.py similarity index 100% rename from siem-converter/app/converter/core/operator_types/__init__.py rename to siem-converter/app/converter/core/custom_types/__init__.py diff --git a/siem-converter/app/converter/core/operator_types/tokens.py b/siem-converter/app/converter/core/custom_types/tokens.py similarity index 80% rename from siem-converter/app/converter/core/operator_types/tokens.py rename to siem-converter/app/converter/core/custom_types/tokens.py index c491fe1a..74206dca 100644 --- a/siem-converter/app/converter/core/operator_types/tokens.py +++ b/siem-converter/app/converter/core/custom_types/tokens.py @@ -14,7 +14,6 @@ class OperatorType(CustomEnum): GTE = ">=" EQ = "=" NEQ = "!=" - COLON = ":" CONTAINS = "contains" STARTSWITH = "startswith" ENDSWITH = "endswith" @@ -25,8 +24,3 @@ class OperatorType(CustomEnum): class GroupType(CustomEnum): L_PAREN = "(" R_PAREN = ")" - GROUP = "group" - - -class ValidTokens(LogicalOperatorType, OperatorType, GroupType): - pass diff --git a/siem-converter/app/converter/core/mixins/operator.py b/siem-converter/app/converter/core/mixins/operator.py index 39a3e27f..fca93a97 100644 --- a/siem-converter/app/converter/core/mixins/operator.py +++ b/siem-converter/app/converter/core/mixins/operator.py @@ -19,7 +19,7 @@ from typing import Union, List, Tuple from app.converter.core.models.identifier import Identifier -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType class WildCardMixin: diff --git a/siem-converter/app/converter/core/models/field.py b/siem-converter/app/converter/core/models/field.py index e881cf3b..61864ee0 100644 --- a/siem-converter/app/converter/core/models/field.py +++ b/siem-converter/app/converter/core/models/field.py @@ -2,7 +2,7 @@ from app.converter.core.mapping import SourceMapping from app.converter.core.models.identifier import Identifier -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType class Field: diff --git a/siem-converter/app/converter/core/models/functions/table.py b/siem-converter/app/converter/core/models/functions/table.py index c8127df3..d4647682 100644 --- a/siem-converter/app/converter/core/models/functions/table.py +++ b/siem-converter/app/converter/core/models/functions/table.py @@ -1,4 +1,4 @@ -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType class TableField: diff --git a/siem-converter/app/converter/core/models/identifier.py b/siem-converter/app/converter/core/models/identifier.py index c62c6bdb..aa0f6f28 100644 --- a/siem-converter/app/converter/core/models/identifier.py +++ b/siem-converter/app/converter/core/models/identifier.py @@ -1,12 +1,16 @@ from dataclasses import dataclass -from app.converter.core.operator_types.tokens import ValidTokens +from app.converter.core.custom_types.tokens import LogicalOperatorType, OperatorType, GroupType + + +class _IdentifierTokenType(LogicalOperatorType, OperatorType, GroupType): + pass @dataclass class Identifier: def __init__(self, *, token_type: str) -> None: - if token_type not in ValidTokens: + if token_type not in _IdentifierTokenType: raise Exception(f"Unexpected token type: {token_type}") self.token_type = token_type diff --git a/siem-converter/app/converter/core/operator_types/output.py b/siem-converter/app/converter/core/models/parser_output.py similarity index 100% rename from siem-converter/app/converter/core/operator_types/output.py rename to siem-converter/app/converter/core/models/parser_output.py diff --git a/siem-converter/app/converter/core/parser.py b/siem-converter/app/converter/core/parser.py index 778cda88..ec2ff1bc 100644 --- a/siem-converter/app/converter/core/parser.py +++ b/siem-converter/app/converter/core/parser.py @@ -22,7 +22,7 @@ from app.converter.core.mapping import BasePlatformMappings, SourceMapping from app.converter.core.models.field import Field from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer from app.converter.core.tokenizer import QueryTokenizer, TOKEN_TYPE diff --git a/siem-converter/app/converter/core/render.py b/siem-converter/app/converter/core/render.py index cf2c97ba..966c80bd 100644 --- a/siem-converter/app/converter/core/render.py +++ b/siem-converter/app/converter/core/render.py @@ -26,8 +26,8 @@ from app.converter.core.models.field import Field, Keyword from app.converter.core.models.functions.types import ParsedFunctions from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer -from app.converter.core.operator_types.tokens import LogicalOperatorType, OperatorType, GroupType +from app.converter.core.models.parser_output import MetaInfoContainer +from app.converter.core.custom_types.tokens import LogicalOperatorType, OperatorType, GroupType class BaseQueryFieldValue(ABC): diff --git a/siem-converter/app/converter/core/tokenizer.py b/siem-converter/app/converter/core/tokenizer.py index 6dab969c..0ced3889 100644 --- a/siem-converter/app/converter/core/tokenizer.py +++ b/siem-converter/app/converter/core/tokenizer.py @@ -27,8 +27,8 @@ from app.converter.core.mapping import SourceMapping, DEFAULT_MAPPING_NAME, BasePlatformMappings from app.converter.core.models.field import Field, Keyword from app.converter.core.models.identifier import Identifier -from app.converter.core.models.group import GroupType -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.platforms.sigma.models.group import GroupType +from app.converter.core.custom_types.tokens import OperatorType from app.converter.tools.utils import get_match_group TOKEN_TYPE = Union[Field, Keyword, Identifier] diff --git a/siem-converter/app/converter/platforms/athena/parsers/athena.py b/siem-converter/app/converter/platforms/athena/parsers/athena.py index f1cc04cf..e6309d96 100644 --- a/siem-converter/app/converter/platforms/athena/parsers/athena.py +++ b/siem-converter/app/converter/platforms/athena/parsers/athena.py @@ -24,7 +24,7 @@ from app.converter.platforms.athena.tokenizer import AthenaTokenizer from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.parser import Parser -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class AthenaParser(Parser): diff --git a/siem-converter/app/converter/platforms/athena/tokenizer.py b/siem-converter/app/converter/platforms/athena/tokenizer.py index 4ce65d6d..8debdd11 100644 --- a/siem-converter/app/converter/platforms/athena/tokenizer.py +++ b/siem-converter/app/converter/platforms/athena/tokenizer.py @@ -21,7 +21,7 @@ from app.converter.core.models.identifier import Identifier from app.converter.core.tokenizer import QueryTokenizer -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType from app.converter.tools.utils import get_match_group diff --git a/siem-converter/app/converter/platforms/base/lucene/parsers/lucene.py b/siem-converter/app/converter/platforms/base/lucene/parsers/lucene.py index 52ea134e..d0c51284 100644 --- a/siem-converter/app/converter/platforms/base/lucene/parsers/lucene.py +++ b/siem-converter/app/converter/platforms/base/lucene/parsers/lucene.py @@ -21,7 +21,7 @@ from app.converter.platforms.base.lucene.tokenizer import LuceneTokenizer from app.converter.core.parser import Parser -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class LuceneParser(Parser): diff --git a/siem-converter/app/converter/platforms/base/lucene/tokenizer.py b/siem-converter/app/converter/platforms/base/lucene/tokenizer.py index 3afd2316..0ac47881 100644 --- a/siem-converter/app/converter/platforms/base/lucene/tokenizer.py +++ b/siem-converter/app/converter/platforms/base/lucene/tokenizer.py @@ -23,7 +23,7 @@ from app.converter.core.models.field import Keyword, Field from app.converter.core.models.identifier import Identifier from app.converter.core.tokenizer import QueryTokenizer -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType from app.converter.tools.utils import get_match_group diff --git a/siem-converter/app/converter/platforms/base/spl/parsers/spl.py b/siem-converter/app/converter/platforms/base/spl/parsers/spl.py index 6d5afc62..834d612f 100644 --- a/siem-converter/app/converter/platforms/base/spl/parsers/spl.py +++ b/siem-converter/app/converter/platforms/base/spl/parsers/spl.py @@ -22,7 +22,7 @@ from app.converter.platforms.base.spl.tokenizer import SplTokenizer from app.converter.core.models.functions.types import ParsedFunctions from app.converter.core.parser import Parser -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class SplParser(Parser): diff --git a/siem-converter/app/converter/platforms/base/spl/tokenizer.py b/siem-converter/app/converter/platforms/base/spl/tokenizer.py index fc920181..e0207cd7 100644 --- a/siem-converter/app/converter/platforms/base/spl/tokenizer.py +++ b/siem-converter/app/converter/platforms/base/spl/tokenizer.py @@ -20,7 +20,7 @@ from typing import Tuple, Any from app.converter.core.tokenizer import QueryTokenizer -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType from app.converter.tools.utils import get_match_group diff --git a/siem-converter/app/converter/platforms/chronicle/parsers/chronicle.py b/siem-converter/app/converter/platforms/chronicle/parsers/chronicle.py index 97e0183a..9ab4b12d 100644 --- a/siem-converter/app/converter/platforms/chronicle/parsers/chronicle.py +++ b/siem-converter/app/converter/platforms/chronicle/parsers/chronicle.py @@ -23,7 +23,7 @@ from app.converter.platforms.chronicle.tokenizer import ChronicleQueryTokenizer from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.parser import Parser -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class ChronicleParser(Parser): diff --git a/siem-converter/app/converter/platforms/chronicle/parsers/chronicle_rule.py b/siem-converter/app/converter/platforms/chronicle/parsers/chronicle_rule.py index 36c2fd38..fc08dfe7 100644 --- a/siem-converter/app/converter/platforms/chronicle/parsers/chronicle_rule.py +++ b/siem-converter/app/converter/platforms/chronicle/parsers/chronicle_rule.py @@ -25,7 +25,7 @@ from app.converter.core.exceptions.parser import TokenizerGeneralException from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.parser import Parser -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class ChronicleRuleParser(Parser): diff --git a/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py b/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py index 6a1c27b6..1a485f60 100644 --- a/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py +++ b/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py @@ -23,7 +23,7 @@ from app.converter.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer from app.converter.tools.utils import concatenate_str, get_author_str _AUTOGENERATED_TITLE = "Autogenerated Chronicle Security rule" diff --git a/siem-converter/app/converter/platforms/chronicle/tokenizer.py b/siem-converter/app/converter/platforms/chronicle/tokenizer.py index a9427911..618d0704 100644 --- a/siem-converter/app/converter/platforms/chronicle/tokenizer.py +++ b/siem-converter/app/converter/platforms/chronicle/tokenizer.py @@ -21,7 +21,7 @@ from app.converter.core.exceptions.parser import TokenizerGeneralException from app.converter.core.tokenizer import QueryTokenizer -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType from app.converter.tools.utils import get_match_group diff --git a/siem-converter/app/converter/platforms/elasticsearch/parsers/detection_rule.py b/siem-converter/app/converter/platforms/elasticsearch/parsers/detection_rule.py index 57fdcb5d..b8e7d6b1 100644 --- a/siem-converter/app/converter/platforms/elasticsearch/parsers/detection_rule.py +++ b/siem-converter/app/converter/platforms/elasticsearch/parsers/detection_rule.py @@ -22,7 +22,7 @@ from app.converter.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchParser from app.converter.core.mixins.rule import JsonRuleMixin from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class ElasticSearchRuleParser(ElasticSearchParser, JsonRuleMixin): diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py b/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py index a38c7c44..7b6bb031 100644 --- a/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py +++ b/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py @@ -25,7 +25,7 @@ from app.converter.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender, ElasticSearchFieldValue from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer from app.converter.tools.utils import concatenate_str, get_mitre_attack_str diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py b/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py index 1e02c210..a07cac49 100644 --- a/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py +++ b/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py @@ -22,7 +22,7 @@ from app.converter.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender, ElasticSearchFieldValue from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer from app.converter.tools.utils import get_author_str, concatenate_str, get_mitre_attack_str, get_licence_str diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py b/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py index 2150642b..6303b6fe 100644 --- a/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py +++ b/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py @@ -25,7 +25,7 @@ from app.converter.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender, ElasticSearchFieldValue from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer from app.converter.tools.utils import concatenate_str, get_author_str, get_licence_str, get_mitre_attack_str, \ get_rule_id_str, get_references_str diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py b/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py index 0272f471..98f1bc21 100644 --- a/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py +++ b/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py @@ -25,7 +25,7 @@ from app.converter.platforms.elasticsearch.const import XPACK_WATCHER_RULE, xpack_watcher_details from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer from app.converter.tools.utils import concatenate_str, get_author_str, get_licence_str, get_mitre_attack_str diff --git a/siem-converter/app/converter/platforms/logscale/parsers/logscale.py b/siem-converter/app/converter/platforms/logscale/parsers/logscale.py index 13fa877f..d7beb82a 100644 --- a/siem-converter/app/converter/platforms/logscale/parsers/logscale.py +++ b/siem-converter/app/converter/platforms/logscale/parsers/logscale.py @@ -25,7 +25,7 @@ from app.converter.platforms.logscale.tokenizer import LogScaleTokenizer from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.parser import Parser -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class LogScaleParser(Parser): diff --git a/siem-converter/app/converter/platforms/logscale/parsers/logscale_alert.py b/siem-converter/app/converter/platforms/logscale/parsers/logscale_alert.py index 7a6484c7..c2584d84 100644 --- a/siem-converter/app/converter/platforms/logscale/parsers/logscale_alert.py +++ b/siem-converter/app/converter/platforms/logscale/parsers/logscale_alert.py @@ -22,7 +22,7 @@ from app.converter.platforms.logscale.parsers.logscale import LogScaleParser from app.converter.core.mixins.rule import JsonRuleMixin from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class LogScaleAlertParser(LogScaleParser, JsonRuleMixin): diff --git a/siem-converter/app/converter/platforms/logscale/renders/logscale.py b/siem-converter/app/converter/platforms/logscale/renders/logscale.py index da9dcf4b..b01ca8ce 100644 --- a/siem-converter/app/converter/platforms/logscale/renders/logscale.py +++ b/siem-converter/app/converter/platforms/logscale/renders/logscale.py @@ -22,7 +22,7 @@ from app.converter.platforms.logscale.mapping import LogScaleMappings, logscale_mappings from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer from app.converter.core.render import BaseQueryRender, BaseQueryFieldValue diff --git a/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py b/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py index 7e87c267..89341497 100644 --- a/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py +++ b/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py @@ -24,7 +24,7 @@ from app.converter.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer _AUTOGENERATED_TITLE = "Autogenerated Falcon LogScale Alert" diff --git a/siem-converter/app/converter/platforms/logscale/tokenizer.py b/siem-converter/app/converter/platforms/logscale/tokenizer.py index 3c02dcd6..dd665c7b 100644 --- a/siem-converter/app/converter/platforms/logscale/tokenizer.py +++ b/siem-converter/app/converter/platforms/logscale/tokenizer.py @@ -21,7 +21,7 @@ from app.converter.core.models.field import Keyword, Field from app.converter.core.models.identifier import Identifier -from app.converter.core.operator_types.tokens import GroupType, LogicalOperatorType, OperatorType +from app.converter.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.converter.core.tokenizer import QueryTokenizer from app.converter.tools.utils import get_match_group diff --git a/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel.py b/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel.py index 32e210d4..dac5851e 100644 --- a/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel.py +++ b/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel.py @@ -25,7 +25,7 @@ from app.converter.core.models.functions.types import ParsedFunctions from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.parser import Parser -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class MicrosoftParser(Parser): diff --git a/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel_rule.py b/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel_rule.py index 156914ac..5b884140 100644 --- a/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel_rule.py +++ b/siem-converter/app/converter/platforms/microsoft/parsers/microsoft_sentinel_rule.py @@ -22,7 +22,7 @@ from app.converter.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftParser from app.converter.core.mixins.rule import JsonRuleMixin from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class MicrosoftRuleParser(MicrosoftParser, JsonRuleMixin): diff --git a/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py b/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py index 78ad74c7..80c69551 100644 --- a/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -19,7 +19,6 @@ import copy import json -import re from app.converter.platforms.microsoft.renders.microsoft_sentinel import ( MicrosoftSentinelQueryRender, @@ -28,7 +27,7 @@ from app.converter.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer from app.converter.tools.utils import concatenate_str, get_author_str, get_licence_str diff --git a/siem-converter/app/converter/platforms/microsoft/tokenizer.py b/siem-converter/app/converter/platforms/microsoft/tokenizer.py index 4eafb3f6..e0f57842 100644 --- a/siem-converter/app/converter/platforms/microsoft/tokenizer.py +++ b/siem-converter/app/converter/platforms/microsoft/tokenizer.py @@ -21,7 +21,7 @@ from app.converter.core.mixins.operator import OperatorBasedMixin from app.converter.core.tokenizer import QueryTokenizer -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType from app.converter.tools.utils import get_match_group diff --git a/siem-converter/app/converter/platforms/opensearch/renders/opensearch_rule.py b/siem-converter/app/converter/platforms/opensearch/renders/opensearch_rule.py index 7c24d0a8..38fc8f11 100644 --- a/siem-converter/app/converter/platforms/opensearch/renders/opensearch_rule.py +++ b/siem-converter/app/converter/platforms/opensearch/renders/opensearch_rule.py @@ -25,7 +25,7 @@ from app.converter.platforms.opensearch.renders.opensearch import OpenSearchQueryRender, OpenSearchFieldValue from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer SEVERITIES_MAP = {"informational": "5", "low": "4", "medium": "3", "high": "2", "critical": "1"} diff --git a/siem-converter/app/converter/platforms/qradar/parsers/qradar.py b/siem-converter/app/converter/platforms/qradar/parsers/qradar.py index e9135e61..3382ecb3 100644 --- a/siem-converter/app/converter/platforms/qradar/parsers/qradar.py +++ b/siem-converter/app/converter/platforms/qradar/parsers/qradar.py @@ -25,7 +25,7 @@ from app.converter.platforms.qradar.tokenizer import QradarTokenizer from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.parser import Parser -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer from app.converter.tools.utils import get_match_group diff --git a/siem-converter/app/converter/platforms/qradar/tokenizer.py b/siem-converter/app/converter/platforms/qradar/tokenizer.py index 680d6a0b..fe36f8ad 100644 --- a/siem-converter/app/converter/platforms/qradar/tokenizer.py +++ b/siem-converter/app/converter/platforms/qradar/tokenizer.py @@ -23,7 +23,7 @@ from app.converter.core.models.field import Keyword from app.converter.core.models.identifier import Identifier from app.converter.core.tokenizer import QueryTokenizer -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType from app.converter.tools.utils import get_match_group diff --git a/siem-converter/app/converter/platforms/roota/parsers/roota.py b/siem-converter/app/converter/platforms/roota/parsers/roota.py index c7690d64..28519cb8 100644 --- a/siem-converter/app/converter/platforms/roota/parsers/roota.py +++ b/siem-converter/app/converter/platforms/roota/parsers/roota.py @@ -18,7 +18,7 @@ from app.converter.core.exceptions.core import UnsupportedRootAParser, RootARuleValidationException from app.converter.core.mixins.rule import YamlRuleMixin -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer from app.converter.core.parser import Parser from app.converter.managers import parser_manager diff --git a/siem-converter/app/converter/core/compiler.py b/siem-converter/app/converter/platforms/sigma/models/compiler.py similarity index 92% rename from siem-converter/app/converter/core/compiler.py rename to siem-converter/app/converter/platforms/sigma/models/compiler.py index e7522b92..a69c948c 100644 --- a/siem-converter/app/converter/core/compiler.py +++ b/siem-converter/app/converter/platforms/sigma/models/compiler.py @@ -17,10 +17,10 @@ """ from app.converter.core.models.field import Field, Keyword -from app.converter.core.models.group import Group +from app.converter.platforms.sigma.models.group import Group from app.converter.core.models.identifier import Identifier -from app.converter.core.models.operator import Operator, NOT -from app.converter.core.operator_types.tokens import LogicalOperatorType, GroupType +from app.converter.platforms.sigma.models.operator import Operator, NOT +from app.converter.core.custom_types.tokens import LogicalOperatorType, GroupType class DataStructureCompiler: diff --git a/siem-converter/app/converter/core/models/group.py b/siem-converter/app/converter/platforms/sigma/models/group.py similarity index 93% rename from siem-converter/app/converter/core/models/group.py rename to siem-converter/app/converter/platforms/sigma/models/group.py index d78eb8e8..b61c50d4 100644 --- a/siem-converter/app/converter/core/models/group.py +++ b/siem-converter/app/converter/platforms/sigma/models/group.py @@ -1,9 +1,7 @@ -from app.converter.core.models.operator import OR, AND, NOT -from app.converter.core.operator_types.tokens import GroupType +from app.converter.platforms.sigma.models.operator import OR, AND, NOT class Group: - token_type = GroupType.GROUP parent_group = [] sub_group = None last_field = None diff --git a/siem-converter/app/converter/platforms/sigma/models/modifiers.py b/siem-converter/app/converter/platforms/sigma/models/modifiers.py index 847258ce..6d1f1d83 100644 --- a/siem-converter/app/converter/platforms/sigma/models/modifiers.py +++ b/siem-converter/app/converter/platforms/sigma/models/modifiers.py @@ -2,7 +2,7 @@ from app.converter.core.models.field import Field from app.converter.core.models.identifier import Identifier -from app.converter.core.operator_types.tokens import LogicalOperatorType, OperatorType, GroupType +from app.converter.core.custom_types.tokens import LogicalOperatorType, OperatorType, GroupType class ModifierManager: diff --git a/siem-converter/app/converter/core/models/operator.py b/siem-converter/app/converter/platforms/sigma/models/operator.py similarity index 93% rename from siem-converter/app/converter/core/models/operator.py rename to siem-converter/app/converter/platforms/sigma/models/operator.py index c59c24f2..7c60f4d9 100644 --- a/siem-converter/app/converter/core/models/operator.py +++ b/siem-converter/app/converter/platforms/sigma/models/operator.py @@ -1,6 +1,6 @@ from abc import ABC -from app.converter.core.operator_types.tokens import LogicalOperatorType +from app.converter.core.custom_types.tokens import LogicalOperatorType class BaseOperator(ABC): diff --git a/siem-converter/app/converter/platforms/sigma/parsers/sigma.py b/siem-converter/app/converter/platforms/sigma/parsers/sigma.py index a9635a38..b41a38f8 100644 --- a/siem-converter/app/converter/platforms/sigma/parsers/sigma.py +++ b/siem-converter/app/converter/platforms/sigma/parsers/sigma.py @@ -28,7 +28,7 @@ from app.converter.core.mixins.rule import YamlRuleMixin from app.converter.core.models.field import Field from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class SigmaParser(YamlRuleMixin): diff --git a/siem-converter/app/converter/platforms/sigma/renders/sigma.py b/siem-converter/app/converter/platforms/sigma/renders/sigma.py index 2ade383d..a1dd5c17 100644 --- a/siem-converter/app/converter/platforms/sigma/renders/sigma.py +++ b/siem-converter/app/converter/platforms/sigma/renders/sigma.py @@ -23,16 +23,16 @@ from app.converter.platforms.sigma.const import SIGMA_RULE_DETAILS from app.converter.platforms.sigma.mapping import SigmaMappings, sigma_mappings, SigmaLogSourceSignature -from app.converter.core.compiler import DataStructureCompiler +from app.converter.platforms.sigma.models.compiler import DataStructureCompiler from app.converter.core.exceptions.core import StrictPlatformFieldException from app.converter.core.mapping import SourceMapping, DEFAULT_MAPPING_NAME from app.converter.core.models.field import Field, Keyword from app.converter.core.models.functions.types import ParsedFunctions -from app.converter.core.models.group import Group -from app.converter.core.models.operator import OR, AND, NOT +from app.converter.platforms.sigma.models.group import Group +from app.converter.platforms.sigma.models.operator import OR, AND, NOT from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer -from app.converter.core.operator_types.tokens import OperatorType +from app.converter.core.models.parser_output import MetaInfoContainer +from app.converter.core.custom_types.tokens import OperatorType class SigmaRender: diff --git a/siem-converter/app/converter/platforms/sigma/tokenizer.py b/siem-converter/app/converter/platforms/sigma/tokenizer.py index 9d16da6c..b73e8515 100644 --- a/siem-converter/app/converter/platforms/sigma/tokenizer.py +++ b/siem-converter/app/converter/platforms/sigma/tokenizer.py @@ -23,7 +23,7 @@ from app.converter.core.exceptions.parser import TokenizerGeneralException from app.converter.core.models.field import Field, Keyword from app.converter.core.models.identifier import Identifier -from app.converter.core.operator_types.tokens import GroupType, LogicalOperatorType +from app.converter.core.custom_types.tokens import GroupType, LogicalOperatorType from app.converter.core.tokenizer import QueryTokenizer diff --git a/siem-converter/app/converter/platforms/splunk/parsers/splunk_alert.py b/siem-converter/app/converter/platforms/splunk/parsers/splunk_alert.py index 3763720a..74fab0b0 100644 --- a/siem-converter/app/converter/platforms/splunk/parsers/splunk_alert.py +++ b/siem-converter/app/converter/platforms/splunk/parsers/splunk_alert.py @@ -22,7 +22,7 @@ from app.converter.platforms.splunk.const import splunk_alert_details from app.converter.platforms.splunk.parsers.splunk import SplunkParser from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import SiemContainer, MetaInfoContainer +from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer class SplunkAlertParser(SplunkParser): diff --git a/siem-converter/app/converter/platforms/splunk/renders/splunk_alert.py b/siem-converter/app/converter/platforms/splunk/renders/splunk_alert.py index f16b9efd..dc80a8e3 100644 --- a/siem-converter/app/converter/platforms/splunk/renders/splunk_alert.py +++ b/siem-converter/app/converter/platforms/splunk/renders/splunk_alert.py @@ -21,7 +21,7 @@ from app.converter.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails -from app.converter.core.operator_types.output import MetaInfoContainer +from app.converter.core.models.parser_output import MetaInfoContainer _AUTOGENERATED_TITLE = "Autogenerated Splunk Alert" From b0586c34430291a622f85c649629e8b3eca10f2b Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Tue, 28 Nov 2023 14:00:17 +0200 Subject: [PATCH 002/542] fix import --- siem-converter/app/converter/core/tokenizer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/siem-converter/app/converter/core/tokenizer.py b/siem-converter/app/converter/core/tokenizer.py index 0ced3889..9e41474a 100644 --- a/siem-converter/app/converter/core/tokenizer.py +++ b/siem-converter/app/converter/core/tokenizer.py @@ -27,8 +27,7 @@ from app.converter.core.mapping import SourceMapping, DEFAULT_MAPPING_NAME, BasePlatformMappings from app.converter.core.models.field import Field, Keyword from app.converter.core.models.identifier import Identifier -from app.converter.platforms.sigma.models.group import GroupType -from app.converter.core.custom_types.tokens import OperatorType +from app.converter.core.custom_types.tokens import OperatorType, GroupType from app.converter.tools.utils import get_match_group TOKEN_TYPE = Union[Field, Keyword, Identifier] From f72ca55d65c8080d34d78248c0f28a37bda5337c Mon Sep 17 00:00:00 2001 From: vh Date: Mon, 4 Dec 2023 12:05:18 +0200 Subject: [PATCH 003/542] Interpret a space as and --- .../app/converter/core/mixins/logic.py | 27 +++++++++++++++++++ .../platforms/base/lucene/tokenizer.py | 7 ++++- .../converter/platforms/base/spl/tokenizer.py | 11 ++++++-- .../converter/platforms/logscale/tokenizer.py | 9 ++++--- 4 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 siem-converter/app/converter/core/mixins/logic.py diff --git a/siem-converter/app/converter/core/mixins/logic.py b/siem-converter/app/converter/core/mixins/logic.py new file mode 100644 index 00000000..84b26a8e --- /dev/null +++ b/siem-converter/app/converter/core/mixins/logic.py @@ -0,0 +1,27 @@ +from typing import List, Union + +from app.converter.core.models.field import Field, Keyword +from app.converter.core.models.identifier import Identifier +from app.converter.core.custom_types.tokens import LogicalOperatorType, GroupType + + +class ANDLogicOperatorMixin: + + @staticmethod + def get_missed_and_token_indices(tokens: List[Union[Field, Keyword, Identifier]]) -> List[int]: + missed_and_indices = [] + for index in range(len(tokens) - 1): + token = tokens[index] + next_token = tokens[index + 1] + if (isinstance(token, (Field, Keyword)) + and not (isinstance(next_token, Identifier) and ( + next_token.token_type in LogicalOperatorType + or next_token.token_type == GroupType.R_PAREN))): + missed_and_indices.append(index + 1) + return reversed(missed_and_indices) + + def add_and_token_if_missed(self, tokens: List[Union[Field, Keyword, Identifier]]) -> List[Union[Field, Keyword, Identifier]]: + indices = self.get_missed_and_token_indices(tokens=tokens) + for index in indices: + tokens.insert(index, Identifier(token_type=LogicalOperatorType.AND)) + return tokens diff --git a/siem-converter/app/converter/platforms/base/lucene/tokenizer.py b/siem-converter/app/converter/platforms/base/lucene/tokenizer.py index 0ac47881..d48acfb5 100644 --- a/siem-converter/app/converter/platforms/base/lucene/tokenizer.py +++ b/siem-converter/app/converter/platforms/base/lucene/tokenizer.py @@ -20,6 +20,7 @@ from typing import Tuple, Union, List, Any from app.converter.core.exceptions.parser import TokenizerGeneralException +from app.converter.core.mixins.logic import ANDLogicOperatorMixin from app.converter.core.models.field import Keyword, Field from app.converter.core.models.identifier import Identifier from app.converter.core.tokenizer import QueryTokenizer @@ -27,7 +28,7 @@ from app.converter.tools.utils import get_match_group -class LuceneTokenizer(QueryTokenizer): +class LuceneTokenizer(QueryTokenizer, ANDLogicOperatorMixin): field_pattern = r"(?P[a-zA-Z\.\-_]+)" match_operator_pattern = r"(?:___field___\s*(?P:))\s*" @@ -107,3 +108,7 @@ def search_keyword(self, query: str) -> Tuple[Keyword, str]: keyword = Keyword(value=value) pos = keyword_search.end() - 1 return keyword, query[pos:] + + def tokenize(self, query: str) -> List[Union[Field, Keyword, Identifier]]: + tokens = super().tokenize(query=query) + return self.add_and_token_if_missed(tokens=tokens) diff --git a/siem-converter/app/converter/platforms/base/spl/tokenizer.py b/siem-converter/app/converter/platforms/base/spl/tokenizer.py index e0207cd7..f4f2f127 100644 --- a/siem-converter/app/converter/platforms/base/spl/tokenizer.py +++ b/siem-converter/app/converter/platforms/base/spl/tokenizer.py @@ -17,14 +17,17 @@ """ import re -from typing import Tuple, Any +from typing import Tuple, Any, List, Union +from app.converter.core.mixins.logic import ANDLogicOperatorMixin +from app.converter.core.models.field import Field, Keyword +from app.converter.core.models.identifier import Identifier from app.converter.core.tokenizer import QueryTokenizer from app.converter.core.custom_types.tokens import OperatorType from app.converter.tools.utils import get_match_group -class SplTokenizer(QueryTokenizer): +class SplTokenizer(QueryTokenizer, ANDLogicOperatorMixin): field_pattern = r"(?P[a-zA-Z\.\-_\{\}]+)" num_value_pattern = r"(?P\d+(?:\.\d+)*)\s*" double_quotes_value_pattern = r'"(?P(?:[:a-zA-Z\*0-9=+%#\-_/,;\'\.$&^@!\(\)\{\}\s]|\\\"|\\)*)"\s*' @@ -51,3 +54,7 @@ def get_operator_and_value(self, match: re.Match, operator: str = OperatorType.E return operator, s_q_value return super().get_operator_and_value(match) + + def tokenize(self, query: str) -> List[Union[Field, Keyword, Identifier]]: + tokens = super().tokenize(query=query) + return self.add_and_token_if_missed(tokens=tokens) diff --git a/siem-converter/app/converter/platforms/logscale/tokenizer.py b/siem-converter/app/converter/platforms/logscale/tokenizer.py index dd665c7b..cba94b07 100644 --- a/siem-converter/app/converter/platforms/logscale/tokenizer.py +++ b/siem-converter/app/converter/platforms/logscale/tokenizer.py @@ -17,8 +17,9 @@ """ import re -from typing import Tuple, Any +from typing import Tuple, Any, List, Union +from app.converter.core.mixins.logic import ANDLogicOperatorMixin from app.converter.core.models.field import Keyword, Field from app.converter.core.models.identifier import Identifier from app.converter.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType @@ -26,7 +27,7 @@ from app.converter.tools.utils import get_match_group -class LogScaleTokenizer(QueryTokenizer): +class LogScaleTokenizer(QueryTokenizer, ANDLogicOperatorMixin): match_operator_pattern = r"""(?:___field___\s?(?P=|!=))\s?""" num_value_pattern = r"(?P\d+(?:\.\d+)*)\s*" double_quotes_value_pattern = r'"(?P(?:[:a-zA-Z\*0-9=+%#\-_/,\'\.$&^@!\(\)\{\}\s]|\\\"|\\)*)"\s*' @@ -65,7 +66,7 @@ def __get_identifier(self, query: str) -> (list, str): else: return self.search_field_value(query) - def tokenize(self, query: str) -> list: + def tokenize(self, query: str) -> List[Union[Field, Keyword, Identifier]]: tokenized = [] while query: identifier, query = self.__get_identifier(query=query) @@ -78,4 +79,4 @@ def tokenize(self, query: str) -> list: tokenized.append(Identifier(token_type=LogicalOperatorType.AND)) tokenized.append(identifier) self._validate_parentheses(tokenized) - return tokenized + return self.add_and_token_if_missed(tokens=tokenized) From a5b935f8c18b8775a570b6ca35666bd451671b07 Mon Sep 17 00:00:00 2001 From: "dmytro.tarnopolskyi" Date: Mon, 4 Dec 2023 12:31:33 +0100 Subject: [PATCH 004/542] tags will be translated into empty list instead of null --- siem-converter/app/converter/platforms/roota/parsers/roota.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siem-converter/app/converter/platforms/roota/parsers/roota.py b/siem-converter/app/converter/platforms/roota/parsers/roota.py index 28519cb8..73ae474f 100644 --- a/siem-converter/app/converter/platforms/roota/parsers/roota.py +++ b/siem-converter/app/converter/platforms/roota/parsers/roota.py @@ -36,7 +36,7 @@ def __update_meta_info(meta_info: MetaInfoContainer, rule: dict) -> MetaInfoCont meta_info.id = rule.get("uuid", meta_info.id) meta_info.references = rule.get("references") meta_info.license = rule.get("license", meta_info.license) - meta_info.tags = rule.get("tags") + meta_info.tags = rule.get("tags", meta_info.tags) meta_info.mitre_attack = mitre_attack meta_info.date = rule.get("date", meta_info.date) meta_info.author = rule.get("author", meta_info.author) From f8436f938fe1732b81581f3db223170ab51b2720 Mon Sep 17 00:00:00 2001 From: "dmytro.tarnopolskyi" Date: Mon, 4 Dec 2023 12:46:37 +0100 Subject: [PATCH 005/542] fix null field when translate to sigma --- .../app/converter/platforms/sigma/renders/sigma.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/siem-converter/app/converter/platforms/sigma/renders/sigma.py b/siem-converter/app/converter/platforms/sigma/renders/sigma.py index a1dd5c17..3d793f1a 100644 --- a/siem-converter/app/converter/platforms/sigma/renders/sigma.py +++ b/siem-converter/app/converter/platforms/sigma/renders/sigma.py @@ -174,12 +174,16 @@ def generate_not(self, data: Any, source_mapping: SourceMapping): return not_node @staticmethod - def generate_field(data: Field, source_mapping: SourceMapping): + def map_field(source_mapping: SourceMapping, generic_field_name: str) -> str: + field_name = source_mapping.fields_mapping.get_platform_field_name(generic_field_name) + return field_name or generic_field_name + + def generate_field(self, data: Field, source_mapping: SourceMapping): source_id = source_mapping.source_id generic_field_name = data.generic_names_map[source_id] if not generic_field_name: raise StrictPlatformFieldException(field_name=data.source_name, platform_name="Sigma") - field_name = source_mapping.fields_mapping.get_platform_field_name(generic_field_name) + field_name = self.map_field(source_mapping, generic_field_name) if data.operator.token_type != OperatorType.EQ: field_name = f"{field_name}|{data.operator.token_type}" if isinstance(data.values, list) and len(data.values) == 1 or isinstance(data.values, (str, int)): From de479f32b33d43a26aac49c631c10f8d5a4f9ba0 Mon Sep 17 00:00:00 2001 From: "dmytro.tarnopolskyi" Date: Mon, 4 Dec 2023 13:22:41 +0100 Subject: [PATCH 006/542] fix small bugs which appear while translating --- siem-converter/app/converter/core/parser.py | 3 +++ .../app/converter/platforms/chronicle/const.py | 1 + .../platforms/chronicle/renders/chronicle_rule.py | 1 + .../platforms/elasticsearch/renders/detection_rule.py | 2 +- .../app/converter/platforms/sigma/parsers/sigma.py | 11 +++++++++-- .../app/dictionaries/uncoder_meta_info_roota.json | 1 - 6 files changed, 15 insertions(+), 4 deletions(-) diff --git a/siem-converter/app/converter/core/parser.py b/siem-converter/app/converter/core/parser.py index ec2ff1bc..08d8b143 100644 --- a/siem-converter/app/converter/core/parser.py +++ b/siem-converter/app/converter/core/parser.py @@ -24,6 +24,7 @@ from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.models.parser_output import SiemContainer, MetaInfoContainer from app.converter.core.tokenizer import QueryTokenizer, TOKEN_TYPE +from app.converter.core.exceptions.parser import TokenizerGeneralException class Parser(ABC): @@ -43,6 +44,8 @@ def get_tokens_and_source_mappings(self, query: str, log_sources: Dict[str, List[str]] ) -> Tuple[List[TOKEN_TYPE], List[SourceMapping]]: + if not query: + raise TokenizerGeneralException("Can't translate empty query. Please provide more details") tokens = self.tokenizer.tokenize(query=query) field_tokens = self.tokenizer.filter_tokens(tokens, Field) field_names = [field.source_name for field in field_tokens] diff --git a/siem-converter/app/converter/platforms/chronicle/const.py b/siem-converter/app/converter/platforms/chronicle/const.py index de66c1af..2e332067 100644 --- a/siem-converter/app/converter/platforms/chronicle/const.py +++ b/siem-converter/app/converter/platforms/chronicle/const.py @@ -8,6 +8,7 @@ rule_id = "" status = "" severity = "" + falsepositives = "" events: diff --git a/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py b/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py index 1a485f60..51fffabe 100644 --- a/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py +++ b/siem-converter/app/converter/platforms/chronicle/renders/chronicle_rule.py @@ -95,4 +95,5 @@ def finalize_query(self, prefix: str, query: str, functions: str, meta_info: Met rule = rule.replace("", meta_info.id) rule = rule.replace("", meta_info.severity) rule = rule.replace("", meta_info.status) + rule = rule.replace("", ', '.join(meta_info.false_positives)) return rule diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py b/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py index 7b6bb031..9380d6d3 100644 --- a/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py +++ b/siem-converter/app/converter/platforms/elasticsearch/renders/detection_rule.py @@ -64,7 +64,7 @@ def finalize_query(self, prefix: str, query: str, functions: str, meta_info: Met "tags": meta_info.mitre_attack, "false_positives": meta_info.false_positives }) - rule_str = json.dumps(rule, indent=4, sort_keys=False) + rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) if not_supported_functions: rendered_not_supported = self.render_not_supported_functions(not_supported_functions) return rule_str + rendered_not_supported diff --git a/siem-converter/app/converter/platforms/sigma/parsers/sigma.py b/siem-converter/app/converter/platforms/sigma/parsers/sigma.py index b41a38f8..206a5b5c 100644 --- a/siem-converter/app/converter/platforms/sigma/parsers/sigma.py +++ b/siem-converter/app/converter/platforms/sigma/parsers/sigma.py @@ -19,7 +19,7 @@ import re -from typing import List +from typing import List, Union from app.converter.platforms.sigma.const import SIGMA_RULE_DETAILS from app.converter.platforms.sigma.mapping import SigmaMappings, sigma_mappings @@ -47,9 +47,16 @@ def __parse_mitre_attack(tags: List[str]) -> List[str]: return result + @staticmethod + def __parse_false_positives(false_positives: Union[str, List[str], None]) -> list: + if isinstance(false_positives, str): + return [i.strip() for i in false_positives.split(',')] + return false_positives + def _get_meta_info(self, rule: dict, source_mapping_ids: List[str]) -> MetaInfoContainer: return MetaInfoContainer( title=rule.get("title"), + id_=rule.get('id'), description=rule.get("description"), author=rule.get("author"), date=rule.get("date"), @@ -58,7 +65,7 @@ def _get_meta_info(self, rule: dict, source_mapping_ids: List[str]) -> MetaInfoC mitre_attack=self.__parse_mitre_attack(rule.get("tags", [])), severity=rule.get("level"), status=rule.get("status"), - false_positives=rule.get("falsepositives"), + false_positives=self.__parse_false_positives(rule.get("falsepositives")), source_mapping_ids=source_mapping_ids ) diff --git a/siem-converter/app/dictionaries/uncoder_meta_info_roota.json b/siem-converter/app/dictionaries/uncoder_meta_info_roota.json index 09289f96..0ab2a1a4 100644 --- a/siem-converter/app/dictionaries/uncoder_meta_info_roota.json +++ b/siem-converter/app/dictionaries/uncoder_meta_info_roota.json @@ -55,7 +55,6 @@ { "name": "logscale-lql-query", "description": "Falcon LogScale Query" }, { "name": "mde-kql-query", "description": "Microsoft Defender for Endpoint Query" }, { "name": "qradar-aql-query", "description": "IBM QRadar Query" }, - { "name": "sigma-yml-rule", "description": "Sigma Rule" }, { "name": "athena-sql-query", "description": "AWS Athena Query (Security Lake)" }, { "name": "chronicle-yaral-query", "description": "Chronicle Security Query" } ] From 2d521fc38fa97bb0178a3d014c45248011f4e937 Mon Sep 17 00:00:00 2001 From: "dmytro.tarnopolskyi" Date: Mon, 4 Dec 2023 13:36:53 +0100 Subject: [PATCH 007/542] improve rule description --- .../elasticsearch/renders/elast_alert.py | 15 ++++--- .../platforms/elasticsearch/renders/kibana.py | 17 +++---- .../elasticsearch/renders/xpack_watcher.py | 12 ++--- .../logscale/renders/logscale_alert.py | 11 +++-- .../renders/microsoft_sentinel_rule.py | 11 ++--- .../platforms/splunk/renders/splunk_alert.py | 11 +++-- siem-converter/app/converter/tools/utils.py | 44 ++++++++++++++++--- 7 files changed, 84 insertions(+), 37 deletions(-) diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py b/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py index a07cac49..89a6c4ba 100644 --- a/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py +++ b/siem-converter/app/converter/platforms/elasticsearch/renders/elast_alert.py @@ -23,7 +23,7 @@ from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.models.parser_output import MetaInfoContainer -from app.converter.tools.utils import get_author_str, concatenate_str, get_mitre_attack_str, get_licence_str +from app.converter.tools.utils import get_rule_description_str SEVERITIES_MAP = {"informational": "5", "low": "4", "medium": "3", "high": "2", "critical": "1"} @@ -48,11 +48,14 @@ def finalize_query(self, prefix: str, query: str, functions: str, meta_info: Met source_mapping: SourceMapping = None, not_supported_functions: list = None): query = super().finalize_query(prefix=prefix, query=query, functions=functions, meta_info=meta_info) rule = ELASTICSEARCH_ALERT.replace("", query) - description = concatenate_str(meta_info.description, get_author_str(meta_info.author)) - description = concatenate_str(description, get_licence_str(meta_info.license)) - description = concatenate_str(description, get_mitre_attack_str(meta_info.mitre_attack)) - - rule = rule.replace("", description) + rule = rule.replace( + "", + get_rule_description_str( + description=meta_info.description, + license=meta_info.license, + mitre_attack=meta_info.mitre_attack + ) + ) rule = rule.replace("", meta_info.title) rule = rule.replace("", SEVERITIES_MAP[meta_info.severity]) if not_supported_functions: diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py b/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py index 6303b6fe..ea0fb6b9 100644 --- a/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py +++ b/siem-converter/app/converter/platforms/elasticsearch/renders/kibana.py @@ -26,8 +26,7 @@ from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.models.parser_output import MetaInfoContainer -from app.converter.tools.utils import concatenate_str, get_author_str, get_licence_str, get_mitre_attack_str, \ - get_rule_id_str, get_references_str +from app.converter.tools.utils import get_rule_description_str class KibanaFieldValue(ElasticSearchFieldValue): @@ -49,12 +48,14 @@ def finalize_query(self, prefix: str, query: str, functions: str, meta_info: Met rule = copy.deepcopy(KIBANA_RULE) rule["_source"]["kibanaSavedObjectMeta"]["searchSourceJSON"] = dumped_rule rule["_source"]["title"] = meta_info.title - description = meta_info.description or rule["_source"]["description"] - description = concatenate_str(description, get_author_str(meta_info.author)) - description = concatenate_str(description, get_rule_id_str(meta_info.id)) - description = concatenate_str(description, get_licence_str(meta_info.license)) - description = concatenate_str(description, get_references_str(meta_info.references)) - rule["_source"]["description"] = concatenate_str(description, get_mitre_attack_str(meta_info.mitre_attack)) + rule["_source"]["description"] = get_rule_description_str( + description=meta_info.description or rule["_source"]["description"], + author=meta_info.author, + rule_id=meta_info.id, + license=meta_info.license, + references=meta_info.references, + mitre_attack=meta_info.mitre_attack + ) rule_str = json.dumps(rule, indent=4, sort_keys=False) if not_supported_functions: rendered_not_supported = self.render_not_supported_functions(not_supported_functions) diff --git a/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py b/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py index 98f1bc21..fa91f58c 100644 --- a/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py +++ b/siem-converter/app/converter/platforms/elasticsearch/renders/xpack_watcher.py @@ -26,7 +26,7 @@ from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.models.parser_output import MetaInfoContainer -from app.converter.tools.utils import concatenate_str, get_author_str, get_licence_str, get_mitre_attack_str +from app.converter.tools.utils import get_rule_description_str class XpackWatcherRuleFieldValue(ElasticSearchFieldValue): @@ -43,13 +43,15 @@ def finalize_query(self, prefix: str, query: str, functions: str, meta_info: Met source_mapping: SourceMapping = None, not_supported_functions: list = None): query = super().finalize_query(prefix=prefix, query=query, functions=functions, meta_info=meta_info) rule = copy.deepcopy(XPACK_WATCHER_RULE) - description = concatenate_str(meta_info.description, get_author_str(meta_info.author)) - description = concatenate_str(description, get_licence_str(meta_info.license)) - description = concatenate_str(description, get_mitre_attack_str(meta_info.mitre_attack)) rule["metadata"].update({ "query": query, "title": meta_info.title, - "description": description, + "description": get_rule_description_str( + description=meta_info.description, + author=meta_info.author, + license=meta_info.license, + mitre_attack=meta_info.mitre_attack + ), "tags": meta_info.mitre_attack }) rule["input"]["search"]["request"]["body"]["query"]["bool"]["must"][0]["query_string"]["query"] = query diff --git a/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py b/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py index 89341497..4c5ad106 100644 --- a/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py +++ b/siem-converter/app/converter/platforms/logscale/renders/logscale_alert.py @@ -25,6 +25,7 @@ from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.models.parser_output import MetaInfoContainer +from app.converter.tools.utils import get_rule_description_str _AUTOGENERATED_TITLE = "Autogenerated Falcon LogScale Alert" @@ -44,10 +45,12 @@ def finalize_query(self, prefix: str, query: str, functions: str, meta_info: Met rule = copy.deepcopy(DEFAULT_LOGSCALE_ALERT) rule['query']['queryString'] = query rule['name'] = meta_info.title or _AUTOGENERATED_TITLE - description = f"{meta_info.description}. Author: {meta_info.author}. License: {meta_info.license}." - if meta_info.mitre_attack: - description += f" MITRE ATT&CK: {', '.join(meta_info.mitre_attack)}" - rule['description'] = description + rule['description'] = get_rule_description_str( + description=meta_info.description, + license=meta_info.license, + mitre_attack=meta_info.mitre_attack, + author=meta_info.author + ) json_query = json.dumps(rule, indent=4, sort_keys=False) if not_supported_functions: diff --git a/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py b/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py index 80c69551..ece9a812 100644 --- a/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/siem-converter/app/converter/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -28,7 +28,7 @@ from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.models.parser_output import MetaInfoContainer -from app.converter.tools.utils import concatenate_str, get_author_str, get_licence_str +from app.converter.tools.utils import get_rule_description_str class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): @@ -46,10 +46,11 @@ def finalize_query(self, prefix: str, query: str, functions: str, meta_info: Met rule = copy.deepcopy(DEFAULT_MICROSOFT_SENTINEL_RULE) rule["query"] = query rule["displayName"] = meta_info.title - description = meta_info.description or rule["description"] - description = concatenate_str(description, get_author_str(meta_info.author)) - description = concatenate_str(description, get_licence_str(meta_info.license)) - rule["description"] = description + rule["description"] = get_rule_description_str( + description=meta_info.description or rule["description"], + author=meta_info.author, + license=meta_info.license + ) rule["severity"] = meta_info.severity rule["techniques"] = [el.upper() for el in meta_info.mitre_attack] json_rule = json.dumps(rule, indent=4, sort_keys=False) diff --git a/siem-converter/app/converter/platforms/splunk/renders/splunk_alert.py b/siem-converter/app/converter/platforms/splunk/renders/splunk_alert.py index dc80a8e3..3284db71 100644 --- a/siem-converter/app/converter/platforms/splunk/renders/splunk_alert.py +++ b/siem-converter/app/converter/platforms/splunk/renders/splunk_alert.py @@ -22,6 +22,7 @@ from app.converter.core.mapping import SourceMapping from app.converter.core.models.platform_details import PlatformDetails from app.converter.core.models.parser_output import MetaInfoContainer +from app.converter.tools.utils import get_rule_description_str _AUTOGENERATED_TITLE = "Autogenerated Splunk Alert" @@ -43,11 +44,13 @@ def finalize_query(self, prefix: str, query: str, functions: str, meta_info: Met rule = DEFAULT_SPLUNK_ALERT.replace("", query) rule = rule.replace("", meta_info.title or _AUTOGENERATED_TITLE) rule = rule.replace("", severity_map.get(meta_info.severity, "1")) + rule_description = get_rule_description_str( + description=meta_info.description or 'Autogenerated Splunk Alert.', + license=meta_info.license, + mitre_attack=meta_info.mitre_attack + ) + rule = rule.replace("", rule_description) - description = f"{meta_info.description or 'Autogenerated Splunk Alert'}. License: {meta_info.license}." - if meta_info.mitre_attack: - description += f" MITRE ATT&CK: {', '.join(meta_info.mitre_attack)}" - rule = rule.replace("", description) if not_supported_functions: rendered_not_supported = self.render_not_supported_functions(not_supported_functions) return rule + rendered_not_supported diff --git a/siem-converter/app/converter/tools/utils.py b/siem-converter/app/converter/tools/utils.py index ae632a4b..c6a0f1a8 100644 --- a/siem-converter/app/converter/tools/utils.py +++ b/siem-converter/app/converter/tools/utils.py @@ -14,20 +14,54 @@ def concatenate_str(str1: str, str2: str) -> str: def get_mitre_attack_str(mitre_attack: List[str]) -> str: - return f"MITRE ATT&CK: {', '.join(mitre_attack).upper()}." if mitre_attack else "" + return f"MITRE ATT&CK: {', '.join(mitre_attack).upper()}." def get_author_str(author: str) -> str: - return f"Author: {author}." if author else "" + return f"Author: {author}." -def get_licence_str(licence: str) -> str: - return f"Licence: {licence}." +def get_license_str(license: str) -> str: + license_str = f"License: {license}" + if not license_str.endswith('.'): + license_str += '.' + return license_str +def get_description_str(description: str) -> str: + if not description.endswith('.'): + description += '.' + return description def get_rule_id_str(rule_id: str) -> str: return f"Rule ID: {rule_id}." def get_references_str(references: List[str]) -> str: - return f"References: {', '.join(references)}." if references else "" + return f"References: {', '.join(references)}." + +def get_rule_description_str( + description: str, + author: str = None, + rule_id: str = None, + license: str = None, + mitre_attack: str = None, + references: str = None +) -> str: + description_str = get_description_str(description) + author_str = get_author_str(author) if author else None + rule_id = get_rule_id_str(rule_id) if rule_id else None + license_str = get_license_str(license) if license else None + mitre_attack = get_mitre_attack_str(mitre_attack) if mitre_attack else None + references = get_references_str(references) if references else None + rule_description = description_str + if author_str: + rule_description = concatenate_str(rule_description, author_str) + if rule_id: + rule_description = concatenate_str(rule_description, rule_id) + if license_str: + rule_description = concatenate_str(rule_description, license_str) + if mitre_attack: + rule_description = concatenate_str(rule_description, mitre_attack) + if references: + rule_description = concatenate_str(rule_description, references) + return rule_description From 093ed6ae45641f85984a602ec458578ff7603a16 Mon Sep 17 00:00:00 2001 From: Mykola Zapeka Date: Wed, 6 Dec 2023 10:23:39 +0200 Subject: [PATCH 008/542] Add UI improvements --- uncoder-os/.env.common | 1 + uncoder-os/.gitignore | 1 + uncoder-os/configs/webpack.common.js | 20 ++++++- uncoder-os/configs/webpack.compile.js | 2 +- uncoder-os/package.json | 5 +- uncoder-os/src/App.tsx | 11 ++++ uncoder-os/src/assets/svg/GridIcon.svg | 55 ++++++++++++++++++- uncoder-os/src/assets/svg/LittleDownIcon.svg | 27 +++++++++ .../src/components/Buttons/Button/Button.sass | 5 ++ .../components/Buttons/Button/ButtonLink.tsx | 17 ++++++ .../src/components/Buttons/Button/index.ts | 1 + .../TranslateButton/TranslateButton.sass | 8 +++ .../TranslateButton/TranslateButton.tsx | 19 ++++++- .../TranslateButton/useTranslateButton.ts | 14 ++++- uncoder-os/src/components/Buttons/index.ts | 2 +- .../DropdownIocSettingsMenu.tsx | 10 ++-- .../ReplaceSettingsMenu.tsx | 9 ++- .../ReplaceSettingsMenu/useReplaceSettings.ts | 5 +- .../ErrorBoundaryFallback.sass | 18 ++++++ .../ErrorBoundaryFallback.tsx | 35 ++++++++++++ .../components/ErrorBoundaryFallback/index.ts | 1 + .../InputEditorFileUploadButton.tsx | 9 ++- .../useInputEditorFileUploadButton.ts | 11 ++-- .../IocOneElement/IocOneElement.tsx | 2 +- .../IocsStatistic/IocStatisticWrapper.tsx | 10 +++- .../IocsStatistic/IocsStatistic.tsx | 27 +++++---- .../IocsStatistic/calcIocsInText.ts | 15 +++-- .../IocsStatistic/useIocCountCalculation.ts | 52 ++++++++++++++++++ .../IocsStatistic/useIocsStatistic.ts | 30 +--------- .../SelectSource/Hooks/useSelectSource.ts | 19 ++++--- .../InputTextEditor/InputTextEditor.tsx | 3 +- .../InputTextEditor/useInputEditor.ts | 39 +++++++++---- .../InputTextEditorHeader.tsx | 28 ++++++---- .../TemplateSelectorButton.tsx | 29 ++++++---- .../useTemplateSelector.ts | 15 +++-- .../useInputTextEditorHeader.ts | 16 ++++-- .../OutputTextEditorHeader.tsx | 4 +- .../useOutputTextEditorHeader.ts | 5 +- .../TextEditorHeader/useTextEditorHeader.ts | 7 ++- .../DownloadInputTextButton.tsx | 19 ++++--- .../InputTextEditorMenu.tsx | 35 +++++++----- .../ReplaceSettingsButton.tsx | 32 ++++++----- .../useInputTextEditorMunu.ts | 4 +- .../DownloadOutputTextButton.tsx | 19 ++++--- .../OutputTextEditorMenu.tsx | 35 +++++++----- .../useOutputTextEditorMunu.ts | 4 +- .../TextEditor/Theme/ThemeSocprime.sass | 1 + .../src/components/Tooltip/Tooltip.sass | 35 +----------- uncoder-os/src/components/Tooltip/Tooltip.tsx | 37 +++++++------ .../useDetectParserByText.ts | 19 +++++-- .../hooks/useInputTextWithFilters/index.ts | 1 + .../useInputTextWithFilters.ts | 24 ++++++++ uncoder-os/src/index.tsx | 4 +- .../src/pages/MainPage/Header/Header.sass | 4 ++ .../src/pages/MainPage/Header/Header.tsx | 4 +- .../src/pages/UncoderEditor/UncoderEditor.tsx | 36 +++++++----- .../src/processes/CalculateIocsMessage.ts | 9 +++ uncoder-os/src/processes/enums.ts | 3 + .../processes/workers/calculateIocs.worker.ts | 37 +++++++++++++ uncoder-os/src/reduxData/RootStore.ts | 3 +- .../src/reduxData/inputEditor/inputEditor.ts | 13 ++++- .../src/reduxData/iocSettings/iocSettings.ts | 5 ++ .../reduxData/outputEditor/outputEditor.ts | 16 ++++-- .../src/reduxData/platforms/platforms.ts | 3 +- .../InputTextFiltersType.ts | 12 ++++ .../applyCustomFilterForRawIocs.ts | 13 +++++ .../replace-brackets-with-dot.ts | 13 +++++ .../filtersHandler/replace-hxxp-with-http.ts | 3 + .../applyCustomFilterForRawIocs/index.ts | 1 + .../inputTextFilters.ts | 17 ++++++ uncoder-os/src/tools/getRandomId.ts | 3 + uncoder-os/src/tools/index.ts | 1 + .../src/tools/injectNonePlatformElement.ts | 7 ++- uncoder-os/src/types/editorValueTypes.ts | 6 ++ uncoder-os/yarn.lock | 55 +++++++++++++++++++ 75 files changed, 838 insertions(+), 282 deletions(-) create mode 100644 uncoder-os/.env.common create mode 100644 uncoder-os/src/App.tsx create mode 100644 uncoder-os/src/assets/svg/LittleDownIcon.svg create mode 100644 uncoder-os/src/components/Buttons/Button/ButtonLink.tsx create mode 100644 uncoder-os/src/components/Buttons/TranslateButton/TranslateButton.sass create mode 100644 uncoder-os/src/components/ErrorBoundaryFallback/ErrorBoundaryFallback.sass create mode 100644 uncoder-os/src/components/ErrorBoundaryFallback/ErrorBoundaryFallback.tsx create mode 100644 uncoder-os/src/components/ErrorBoundaryFallback/index.ts create mode 100644 uncoder-os/src/components/IocsStatistic/useIocCountCalculation.ts create mode 100644 uncoder-os/src/hooks/useInputTextWithFilters/index.ts create mode 100644 uncoder-os/src/hooks/useInputTextWithFilters/useInputTextWithFilters.ts create mode 100644 uncoder-os/src/processes/CalculateIocsMessage.ts create mode 100644 uncoder-os/src/processes/enums.ts create mode 100644 uncoder-os/src/processes/workers/calculateIocs.worker.ts create mode 100644 uncoder-os/src/tools/applyCustomFilterForRawIocs/InputTextFiltersType.ts create mode 100644 uncoder-os/src/tools/applyCustomFilterForRawIocs/applyCustomFilterForRawIocs.ts create mode 100644 uncoder-os/src/tools/applyCustomFilterForRawIocs/filtersHandler/replace-brackets-with-dot.ts create mode 100644 uncoder-os/src/tools/applyCustomFilterForRawIocs/filtersHandler/replace-hxxp-with-http.ts create mode 100644 uncoder-os/src/tools/applyCustomFilterForRawIocs/index.ts create mode 100644 uncoder-os/src/tools/applyCustomFilterForRawIocs/inputTextFilters.ts create mode 100644 uncoder-os/src/tools/getRandomId.ts create mode 100644 uncoder-os/src/types/editorValueTypes.ts diff --git a/uncoder-os/.env.common b/uncoder-os/.env.common new file mode 100644 index 00000000..c81b6bf1 --- /dev/null +++ b/uncoder-os/.env.common @@ -0,0 +1 @@ +MODULE_VERSION=1.1.0 diff --git a/uncoder-os/.gitignore b/uncoder-os/.gitignore index 71036779..03988423 100644 --- a/uncoder-os/.gitignore +++ b/uncoder-os/.gitignore @@ -20,3 +20,4 @@ yarn-error.log* # .env /.idea +.editorconfig diff --git a/uncoder-os/configs/webpack.common.js b/uncoder-os/configs/webpack.common.js index f1a2073c..3701f825 100644 --- a/uncoder-os/configs/webpack.common.js +++ b/uncoder-os/configs/webpack.common.js @@ -19,7 +19,25 @@ const rules = [ onlyCompileBundledFiles: true, }, }], - exclude: /node_modules/, + exclude: [/node_modules/, /workers/], + }, + { + test: /\.worker\.ts$/, + use: [ + { + loader: 'worker-loader', + options: { + inline: 'no-fallback', + }, + }, + 'babel-loader', + { + loader: 'ts-loader', + options: { + onlyCompileBundledFiles: true, + }, + }], + exclude: [/node_modules/], }, { test: /\.svg$/, diff --git a/uncoder-os/configs/webpack.compile.js b/uncoder-os/configs/webpack.compile.js index ba139f83..774852a9 100644 --- a/uncoder-os/configs/webpack.compile.js +++ b/uncoder-os/configs/webpack.compile.js @@ -12,7 +12,7 @@ const plugins = [ name: 'uncoder', filename: 'remoteEntry.js', exposes: { - './UncoderEditor': './src/pages/UncoderEditor/bootstrap', + './UncoderEditor': './src/pages/UncoderEditor/bootstrap.tsx', }, shared: packageJson.dependencies, }), diff --git a/uncoder-os/package.json b/uncoder-os/package.json index b9fa8cf7..3723a932 100644 --- a/uncoder-os/package.json +++ b/uncoder-os/package.json @@ -16,7 +16,9 @@ "react": "^18", "react-ace": "^10.1.0", "react-dom": "^18", + "react-error-boundary": "^4.0.11", "react-redux": "^8.1.3", + "react-tooltip": "^5.24.0", "redux": "^4.2.1", "simplebar-react": "^3.2.4" }, @@ -63,6 +65,7 @@ "webpack": "^5.89.0", "webpack-cli": "^5.1.4", "webpack-dev-server": "^4.15.1", - "webpack-merge": "^5.10.0" + "webpack-merge": "^5.10.0", + "worker-loader": "^3.0.8" } } diff --git a/uncoder-os/src/App.tsx b/uncoder-os/src/App.tsx new file mode 100644 index 00000000..1017aa71 --- /dev/null +++ b/uncoder-os/src/App.tsx @@ -0,0 +1,11 @@ +import React, { FC, Suspense } from 'react'; +import { MainPage } from './pages/MainPage'; +import { Spinner } from './components/Spinner'; + +const App: FC = () => ( + }> + + +); + +export default App; diff --git a/uncoder-os/src/assets/svg/GridIcon.svg b/uncoder-os/src/assets/svg/GridIcon.svg index d6c78d63..1195f6c9 100644 --- a/uncoder-os/src/assets/svg/GridIcon.svg +++ b/uncoder-os/src/assets/svg/GridIcon.svg @@ -1 +1,54 @@ - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/uncoder-os/src/assets/svg/LittleDownIcon.svg b/uncoder-os/src/assets/svg/LittleDownIcon.svg new file mode 100644 index 00000000..eb0241c0 --- /dev/null +++ b/uncoder-os/src/assets/svg/LittleDownIcon.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/uncoder-os/src/components/Buttons/Button/Button.sass b/uncoder-os/src/components/Buttons/Button/Button.sass index 46e6ef1d..bff219e8 100644 --- a/uncoder-os/src/components/Buttons/Button/Button.sass +++ b/uncoder-os/src/components/Buttons/Button/Button.sass @@ -14,8 +14,11 @@ font-weight: 400 font-size: 14px color: $textDarkBlue + text-decoration: none transition: background-color .1s cursor: pointer + &:hover + text-decoration: none &:disabled, &.disabled &, @@ -58,3 +61,5 @@ font-size: 16px &.button--icon min-width: 30px + &--inline + display: inline-flex diff --git a/uncoder-os/src/components/Buttons/Button/ButtonLink.tsx b/uncoder-os/src/components/Buttons/Button/ButtonLink.tsx new file mode 100644 index 00000000..4e32db87 --- /dev/null +++ b/uncoder-os/src/components/Buttons/Button/ButtonLink.tsx @@ -0,0 +1,17 @@ +import React, { AnchorHTMLAttributes, FC } from 'react'; + +import './Button.sass'; + +type ButtonLinkPropsType = AnchorHTMLAttributes & { + classes?: string; +}; + +export const ButtonLink: FC = ({ + children, + classes = '', + ...otherProps +}) => ( + + {children} + +); diff --git a/uncoder-os/src/components/Buttons/Button/index.ts b/uncoder-os/src/components/Buttons/Button/index.ts index fe9c53c5..d2896d97 100644 --- a/uncoder-os/src/components/Buttons/Button/index.ts +++ b/uncoder-os/src/components/Buttons/Button/index.ts @@ -1 +1,2 @@ export { Button } from './Button'; +export { ButtonLink } from './ButtonLink'; diff --git a/uncoder-os/src/components/Buttons/TranslateButton/TranslateButton.sass b/uncoder-os/src/components/Buttons/TranslateButton/TranslateButton.sass new file mode 100644 index 00000000..c1bc1e13 --- /dev/null +++ b/uncoder-os/src/components/Buttons/TranslateButton/TranslateButton.sass @@ -0,0 +1,8 @@ +@import ../../../assets/sass/helpers/variables + +.button--upper + &:disabled, &:disabled:hover + border-color: $borderDisabled + background-color: $iconDisabled + box-shadow: none + z-index: 1 \ No newline at end of file diff --git a/uncoder-os/src/components/Buttons/TranslateButton/TranslateButton.tsx b/uncoder-os/src/components/Buttons/TranslateButton/TranslateButton.tsx index 47d53f79..97c5c1ce 100644 --- a/uncoder-os/src/components/Buttons/TranslateButton/TranslateButton.tsx +++ b/uncoder-os/src/components/Buttons/TranslateButton/TranslateButton.tsx @@ -1,9 +1,26 @@ import React, { FC } from 'react'; import { Button } from '../Button'; +import { Tooltip } from '../../Tooltip'; import { useTranslateButton } from './useTranslateButton'; +import './TranslateButton.sass'; + export const TranslateButton: FC = () => { - const { onClickHandler } = useTranslateButton(); + const { isActive, disabledMessage, onClickHandler } = useTranslateButton(); + + if (!isActive) { + return ( + +