Skip to content

platforms init refactoring #107

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
May 10, 2024
Merged

platforms init refactoring #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b63c0ab
gis-7683 automate parser/render collection
nazargesyk Apr 26, 2024
6a53064
Merge branch 'prod' into 'gis-7683'
nazargesyk Apr 26, 2024
9985f6b
gis-7683 automate parser/render collection fixes
nazargesyk Apr 30, 2024
0d189f6
Merge branch 'prod' into gis-7683
nazargesyk Apr 30, 2024
e44aa8e
gis-7683 fixes
nazargesyk Apr 30, 2024
c381cf6
gis-7683 fixes
nazargesyk Apr 30, 2024
40f8c29
resolve conflicts
alexvolha May 8, 2024
4b9e590
resolve conflicts
May 9, 2024
15bd579
add license
May 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions uncoder-core/app/translator/cti_translator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@


class CTITranslator:
renders: RenderCTIManager = render_cti_manager
render_manager: RenderCTIManager = render_cti_manager

def __init__(self):
self.logger = logging.getLogger("cti_translator")
Expand Down Expand Up @@ -38,7 +38,7 @@ def __parse_iocs_from_string(

@handle_translation_exceptions
def __render_translation(self, parsed_data: dict, platform_data: CTIPlatform, iocs_per_query: int) -> list[str]:
render_cti = self.renders.get(platform_data.id)
render_cti = self.render_manager.get(platform_data.id)

chunked_iocs = self.__get_iocs_chunk(
chunks_size=iocs_per_query, data=parsed_data, mapping=render_cti.default_mapping
Expand Down Expand Up @@ -85,4 +85,4 @@ def __get_iocs_chunk(

@classmethod
def get_renders(cls) -> list:
return cls.renders.get_platforms_details
return cls.render_manager.get_platforms_details
66 changes: 51 additions & 15 deletions uncoder-core/app/translator/managers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,26 @@
from abc import ABC
from functools import cached_property

from app.models.translation import TranslatorPlatform
from app.translator.core.exceptions.core import UnsupportedRootAParser
from app.translator.platforms import __ALL_PARSERS as PARSERS
from app.translator.platforms import __ALL_RENDERS as RENDERS
from app.translator.platforms import __ALL_RENDERS_CTI as RENDERS_CTI


class Manager(ABC):
platforms_class = ()
platforms = {}

@property
def platforms(self) -> dict:
return {platform.details.platform_id: platform for platform in self.platforms_class}
def register(self, cls):
self.platforms[cls.details.platform_id] = cls()
return cls

def get(self, platform_id: str): # noqa: ANN201
if platform := self.platforms.get(platform_id):
return platform
raise UnsupportedRootAParser(parser=platform_id)

def all_platforms(self) -> list:
return list(self.platforms)
return list(self.platforms.keys())

@property
@cached_property
def get_platforms_details(self) -> list[TranslatorPlatform]:
platforms = [
TranslatorPlatform(
Expand All @@ -37,21 +35,59 @@ def get_platforms_details(self) -> list[TranslatorPlatform]:
alt_platform=platform.details.alt_platform,
first_choice=platform.details.first_choice,
)
for platform in self.platforms_class
for platform in self.platforms.values()
]
return sorted(platforms, key=lambda platform: platform.group_name)


class RenderManager(Manager):
platforms_class = RENDERS
class ParserManager(Manager):
platforms = {}
supported_by_roota_platforms = {}
main_platforms = {}

def get_supported_by_roota(self, platform_id: str): # noqa: ANN201
if platform := self.supported_by_roota_platforms.get(platform_id):
return platform
raise UnsupportedRootAParser(parser=platform_id)

class ParserManager(Manager):
platforms_class = PARSERS
def register_supported_by_roota(self, cls):
parser = cls()
self.supported_by_roota_platforms[cls.details.platform_id] = parser
self.platforms[cls.details.platform_id] = parser
return cls

def register_main(self, cls):
parser = cls()
self.main_platforms[cls.details.platform_id] = parser
self.platforms[cls.details.platform_id] = parser
return cls

@cached_property
def get_platforms_details(self) -> list[TranslatorPlatform]:
platforms = [
TranslatorPlatform(
id=platform.details.platform_id,
name=platform.details.name,
code=platform.details.platform_id,
group_name=platform.details.group_name,
group_id=platform.details.group_id,
platform_name=platform.details.platform_name,
platform_id=platform.details.platform_id,
alt_platform_name=platform.details.alt_platform_name,
alt_platform=platform.details.alt_platform,
first_choice=platform.details.first_choice,
)
for platform in self.platforms.values()
]
return sorted(platforms, key=lambda platform: platform.group_name)


class RenderManager(Manager):
platforms = {}


class RenderCTIManager(Manager):
platforms_class = RENDERS_CTI
platforms = {}


parser_manager = ParserManager()
Expand Down
143 changes: 13 additions & 130 deletions uncoder-core/app/translator/platforms/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,134 +1,17 @@
from app.translator.platforms.athena.parsers.athena import AthenaQueryParser
from app.translator.platforms.athena.renders.athena import AthenaQueryRender
from app.translator.platforms.athena.renders.athena_cti import AthenaCTI
from app.translator.platforms.carbonblack.renders.carbonblack_cti import CarbonBlackCTI
from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser
from app.translator.platforms.chronicle.parsers.chronicle_rule import ChronicleRuleParser
from app.translator.platforms.chronicle.renders.chronicle import ChronicleQueryRender
from app.translator.platforms.chronicle.renders.chronicle_cti import ChronicleQueryCTI
from app.translator.platforms.chronicle.renders.chronicle_rule import ChronicleSecurityRuleRender
from app.translator.platforms.crowdstrike.parsers.crowdstrike import CrowdStrikeQueryParser
from app.translator.platforms.crowdstrike.renders.crowdstrike import CrowdStrikeQueryRender
from app.translator.platforms.crowdstrike.renders.crowdstrike_cti import CrowdStrikeCTI
from app.translator.platforms.elasticsearch.parsers.detection_rule import ElasticSearchRuleParser
from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser
from app.translator.platforms.elasticsearch.renders.detection_rule import ElasticSearchRuleRender
from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender
from app.translator.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender
from app.translator.platforms.elasticsearch.renders.elasticsearch_cti import ElasticsearchCTI
from app.translator.platforms.elasticsearch.renders.kibana import KibanaRuleRender
from app.translator.platforms.elasticsearch.renders.xpack_watcher import XPackWatcherRuleRender
from app.translator.platforms.fireeye_helix.renders.fireeye_helix_cti import FireeyeHelixCTI
from app.translator.platforms.forti_siem.renders.forti_siem_rule import FortiSiemRuleRender
from app.translator.platforms.graylog.parsers.graylog import GraylogQueryParser
from app.translator.platforms.graylog.renders.graylog import GraylogQueryRender
from app.translator.platforms.graylog.renders.graylog_cti import GraylogCTI
from app.translator.platforms.logpoint.renders.logpoint_cti import LogpointCTI
from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import LogRhythmAxonQueryRender
from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_rule import LogRhythmAxonRuleRender
from app.translator.platforms.logscale.parsers.logscale import LogScaleQueryParser
from app.translator.platforms.logscale.parsers.logscale_alert import LogScaleAlertParser
from app.translator.platforms.logscale.renders.logscale import LogScaleQueryRender
from app.translator.platforms.logscale.renders.logscale_alert import LogScaleAlertRender
from app.translator.platforms.logscale.renders.logscale_cti import LogScaleCTI
from app.translator.platforms.microsoft.parsers.microsoft_defender import MicrosoftDefenderQueryParser
from app.translator.platforms.microsoft.parsers.microsoft_sentinel import MicrosoftSentinelQueryParser
from app.translator.platforms.microsoft.parsers.microsoft_sentinel_rule import MicrosoftSentinelRuleParser
from app.translator.platforms.microsoft.renders.microsoft_defender import MicrosoftDefenderQueryRender
from app.translator.platforms.microsoft.renders.microsoft_defender_cti import MicrosoftDefenderCTI
from app.translator.platforms.microsoft.renders.microsoft_sentinel import MicrosoftSentinelQueryRender
from app.translator.platforms.microsoft.renders.microsoft_sentinel_cti import MicrosoftSentinelCTI
from app.translator.platforms.microsoft.renders.microsoft_sentinel_rule import MicrosoftSentinelRuleRender
from app.translator.platforms.opensearch.parsers.opensearch import OpenSearchQueryParser
from app.translator.platforms.opensearch.renders.opensearch import OpenSearchQueryRender
from app.translator.platforms.opensearch.renders.opensearch_cti import OpenSearchCTI
from app.translator.platforms.opensearch.renders.opensearch_rule import OpenSearchRuleRender
from app.translator.platforms.palo_alto.renders.cortex_xsiam import CortexXQLQueryRender
from app.translator.platforms.qradar.parsers.qradar import QradarQueryParser
from app.translator.platforms.qradar.renders.qradar import QradarQueryRender
from app.translator.platforms.qradar.renders.qradar_cti import QRadarCTI
from app.translator.platforms.qualys.renders.qualys_cti import QualysCTI
from app.translator.platforms.rsa_netwitness.renders.rsa_netwitness_cti import RSANetwitnessCTI
from app.translator.platforms.securonix.renders.securonix_cti import SecuronixCTI
from app.translator.platforms.sentinel_one.renders.s1_cti import S1EventsCTI
from app.translator.platforms.sigma.parsers.sigma import SigmaParser
from app.translator.platforms.sigma.renders.sigma import SigmaRender
from app.translator.platforms.snowflake.renders.snowflake_cti import SnowflakeCTI
from app.translator.platforms.splunk.parsers.splunk import SplunkQueryParser
from app.translator.platforms.splunk.parsers.splunk_alert import SplunkAlertParser
from app.translator.platforms.splunk.renders.splunk import SplunkQueryRender
from app.translator.platforms.splunk.renders.splunk_alert import SplunkAlertRender
from app.translator.platforms.splunk.renders.splunk_cti import SplunkCTI
from app.translator.platforms.sumo_logic.renders.sumologic_cti import SumologicCTI
import importlib.util
import os

__ALL_RENDERS = (
SigmaRender(),
MicrosoftSentinelQueryRender(),
MicrosoftSentinelRuleRender(),
MicrosoftDefenderQueryRender(),
QradarQueryRender(),
CrowdStrikeQueryRender(),
SplunkQueryRender(),
SplunkAlertRender(),
ChronicleQueryRender(),
ChronicleSecurityRuleRender(),
AthenaQueryRender(),
ElasticSearchQueryRender(),
LogRhythmAxonQueryRender(),
LogRhythmAxonRuleRender(),
LogScaleQueryRender(),
LogScaleAlertRender(),
ElasticSearchRuleRender(),
ElastAlertRuleRender(),
KibanaRuleRender(),
XPackWatcherRuleRender(),
OpenSearchQueryRender(),
OpenSearchRuleRender(),
GraylogQueryRender(),
FortiSiemRuleRender(),
CortexXQLQueryRender(),
)
from const import PLATFORMS_PATH

__ALL_PARSERS = (
AthenaQueryParser(),
ChronicleQueryParser(),
ChronicleRuleParser(),
SplunkQueryParser(),
SplunkAlertParser(),
SigmaParser(),
QradarQueryParser(),
MicrosoftSentinelQueryParser(),
MicrosoftSentinelRuleParser(),
MicrosoftDefenderQueryParser(),
CrowdStrikeQueryParser(),
LogScaleQueryParser(),
LogScaleAlertParser(),
ElasticSearchQueryParser(),
ElasticSearchRuleParser(),
OpenSearchQueryParser(),
GraylogQueryParser(),
)

def init_platforms():
for platform in [f for f in os.listdir(PLATFORMS_PATH) if os.path.isdir(os.path.join(PLATFORMS_PATH, f))]:
if not platform.startswith("__") and not platform.endswith("__"):
# Platforms __init__.py execution
init_path = f"{PLATFORMS_PATH}/{platform}/__init__.py"
spec = importlib.util.spec_from_file_location("__init__", init_path)
init_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(init_module)

__ALL_RENDERS_CTI = (
MicrosoftSentinelCTI(),
MicrosoftDefenderCTI(),
QRadarCTI(),
SplunkCTI(),
ChronicleQueryCTI(),
CrowdStrikeCTI(),
SumologicCTI(),
ElasticsearchCTI(),
LogScaleCTI(),
OpenSearchCTI(),
FireeyeHelixCTI(),
CarbonBlackCTI(),
GraylogCTI(),
LogpointCTI(),
QualysCTI(),
RSANetwitnessCTI(),
S1EventsCTI(),
SecuronixCTI(),
SnowflakeCTI(),
AthenaCTI(),
)

init_platforms()
1 change: 1 addition & 0 deletions uncoder-core/app/translator/platforms/arcsight/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from app.translator.platforms.arcsight.renders.arcsight_cti import ArcsightKeyword
19 changes: 19 additions & 0 deletions uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.render_cti import RenderCTI
from app.translator.managers import render_cti_manager
from app.translator.platforms.arcsight.const import ARCSIGHT_QUERY_DETAILS
from app.translator.platforms.arcsight.mappings.arcsight_cti import DEFAULT_ARCSIGHT_MAPPING


@render_cti_manager.register
class ArcsightKeyword(RenderCTI):
details: PlatformDetails = PlatformDetails(**ARCSIGHT_QUERY_DETAILS)

default_mapping = DEFAULT_ARCSIGHT_MAPPING
field_value_template: str = "{key} = {value}"
or_operator: str = " OR "
group_or_operator: str = " OR "
or_group: str = "{or_group}"
result_join: str = ""
final_result_for_many: str = '({result}) AND type != 2 | rex field = flexString1 mode=sed "s//Sigma: None/g"\n'
final_result_for_one: str = '{result} AND type != 2 | rex field = flexString1 mode=sed "s//Sigma: None/g"\n'
3 changes: 3 additions & 0 deletions uncoder-core/app/translator/platforms/athena/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from app.translator.platforms.athena.parsers.athena import AthenaQueryParser
from app.translator.platforms.athena.renders.athena import AthenaQueryRender
from app.translator.platforms.athena.renders.athena_cti import AthenaCTI
2 changes: 2 additions & 0 deletions uncoder-core/app/translator/platforms/athena/parsers/athena.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,13 @@
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
from app.translator.core.parser import PlatformQueryParser
from app.translator.managers import parser_manager
from app.translator.platforms.athena.const import athena_details
from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings
from app.translator.platforms.athena.tokenizer import AthenaTokenizer


@parser_manager.register_supported_by_roota
class AthenaQueryParser(PlatformQueryParser):
details: PlatformDetails = athena_details
mappings: AthenaMappings = athena_mappings
Expand Down
2 changes: 2 additions & 0 deletions uncoder-core/app/translator/platforms/athena/renders/athena.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from app.translator.core.mapping import LogSourceSignature
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender
from app.translator.managers import render_manager
from app.translator.platforms.athena.const import athena_details
from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings

Expand Down Expand Up @@ -76,6 +77,7 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: # noqa: ARG00
raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords")


@render_manager.register
class AthenaQueryRender(PlatformQueryRender):
details: PlatformDetails = athena_details
mappings: AthenaMappings = athena_mappings
Expand Down
2 changes: 2 additions & 0 deletions uncoder-core/app/translator/platforms/athena/renders/athena_cti.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@

from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.render_cti import RenderCTI
from app.translator.managers import render_cti_manager
from app.translator.platforms.athena.const import athena_details
from app.translator.platforms.athena.mappings.athena_cti import DEFAULT_ATHENA_MAPPING


@render_cti_manager.register
class AthenaCTI(RenderCTI):
details: PlatformDetails = athena_details

Expand Down
1 change: 1 addition & 0 deletions uncoder-core/app/translator/platforms/carbonblack/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from app.translator.platforms.carbonblack.renders.carbonblack_cti import CarbonBlackCTI
2 changes: 2 additions & 0 deletions uncoder-core/app/translator/platforms/carbonblack/renders/carbonblack_cti.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@

from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.render_cti import RenderCTI
from app.translator.managers import render_cti_manager
from app.translator.platforms.carbonblack.const import CARBON_BLACK_QUERY_DETAILS
from app.translator.platforms.carbonblack.mappings.carbonblack_cti import DEFAULT_CARBONBLACK_MAPPING


@render_cti_manager.register
class CarbonBlackCTI(RenderCTI):
details: PlatformDetails = PlatformDetails(**CARBON_BLACK_QUERY_DETAILS)

Expand Down
5 changes: 5 additions & 0 deletions uncoder-core/app/translator/platforms/chronicle/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser
from app.translator.platforms.chronicle.parsers.chronicle_rule import ChronicleRuleParser
from app.translator.platforms.chronicle.renders.chronicle import ChronicleQueryRender
from app.translator.platforms.chronicle.renders.chronicle_cti import ChronicleQueryCTI
from app.translator.platforms.chronicle.renders.chronicle_rule import ChronicleSecurityRuleRender
2 changes: 2 additions & 0 deletions uncoder-core/app/translator/platforms/chronicle/parsers/chronicle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
from app.translator.core.parser import PlatformQueryParser
from app.translator.managers import parser_manager
from app.translator.platforms.chronicle.const import chronicle_query_details
from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings
from app.translator.platforms.chronicle.tokenizer import ChronicleQueryTokenizer


@parser_manager.register_supported_by_roota
class ChronicleQueryParser(PlatformQueryParser):
mappings: ChronicleMappings = chronicle_mappings
tokenizer: ChronicleQueryTokenizer = ChronicleQueryTokenizer()
Expand Down
2 changes: 2 additions & 0 deletions uncoder-core/app/translator/platforms/chronicle/parsers/chronicle_rule.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@
from app.translator.core.exceptions.parser import TokenizerGeneralException
from app.translator.core.models.platform_details import PlatformDetails
from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer
from app.translator.managers import parser_manager
from app.translator.platforms.chronicle.const import chronicle_rule_details
from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings
from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser
from app.translator.platforms.chronicle.tokenizer import ChronicleRuleTokenizer


@parser_manager.register
class ChronicleRuleParser(ChronicleQueryParser):
details: PlatformDetails = chronicle_rule_details
rule_name_pattern = "rule\s(?P<rule_name>[a-z0-9_]+)\s{"
Expand Down
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy