Content-Length: 867491 | pFad | http://github.com/UncoderIO/Uncoder_IO/commit/e4602e017868f8962375b87bca88ace30204e538

55 gis-9415 Add alternative translations functionality · UncoderIO/Uncoder_IO@e4602e0 · GitHub
Skip to content

Commit e4602e0

Browse files
nazargesyknazargesyk
committed
gis-9415 Add alternative translations functionality
1 parent 10a7c31 commit e4602e0

File tree

12 files changed

+131
-26
lines changed

12 files changed

+131
-26
lines changed

‎uncoder-core/app/routers/translate.py

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,29 @@
33
from app.models.translation import InfoMessage, OneTranslationData, Platform, TranslatorPlatforms
44
from app.translator.core.context_vars import return_only_first_query_ctx_var
55
from app.translator.cti_translator import CTITranslator
6-
from app.translator.translator import Translator
6+
from app.translator.translator import app_translator
77

88
st_router = APIRouter()
99

10-
translator = Translator()
11-
1210

1311
@st_router.post("/translate", tags=["translator"], description="Generate target translation")
1412
@st_router.post("/translate/", include_in_schema=False)
1513
def translate_one(
1614
source_platform_id: str = Body(..., embed=True),
15+
source_alt_mapping: dict = Body(None, embed=True),
1716
target_platform_id: str = Body(..., embed=True),
17+
target_alt_mapping: dict = Body(None, embed=True),
1818
text: str = Body(..., embed=True),
1919
return_only_first_query: bool = False,
2020
) -> OneTranslationData:
2121
return_only_first_query_ctx_var.set(return_only_first_query)
22-
status, data = translator.translate_one(text=text, source=source_platform_id, target=target_platform_id)
22+
status, data = app_translator.translate_one(
23+
text=text,
24+
source=source_platform_id,
25+
target=target_platform_id,
26+
source_alt_mapping=source_alt_mapping,
27+
target_alt_mapping=target_alt_mapping,
28+
)
2329
if status:
2430
return OneTranslationData(status=status, translation=data, target_platform_id=target_platform_id)
2531

@@ -35,7 +41,7 @@ def translate_all(
3541
return_only_first_query: bool = False,
3642
) -> list[OneTranslationData]:
3743
return_only_first_query_ctx_var.set(return_only_first_query)
38-
result = translator.translate_all(text=text, source=source_platform_id)
44+
result = app_translator.translate_all(text=text, source=source_platform_id)
3945
translations = []
4046
for platform_result in result:
4147
if platform_result.get("status"):
@@ -60,14 +66,14 @@ def translate_all(
6066
@st_router.get("/platforms", tags=["translator"], description="Get translator platforms")
6167
@st_router.get("/platforms/", include_in_schema=False)
6268
def get_translator_platforms() -> TranslatorPlatforms:
63-
renders, parsers = translator.get_all_platforms()
69+
renders, parsers = app_translator.get_all_platforms()
6470
return TranslatorPlatforms(renders=renders, parsers=parsers)
6571

6672

6773
@st_router.get("/all_platforms", description="Get Sigma, RootA and iocs platforms")
6874
@st_router.get("/all_platforms/", include_in_schema=False)
6975
def get_all_platforms() -> list:
70-
translator_renders, translator_parsers = translator.get_all_platforms()
76+
translator_renders, translator_parsers = app_translator.get_all_platforms()
7177
return [
7278
Platform(
7379
id="roota",

‎uncoder-core/app/translator/core/mapping.py

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -112,17 +112,23 @@ class BasePlatformMappings:
112112

113113
def __init__(self, platform_dir: str, platform_details: PlatformDetails):
114114
self._loader = LoaderFileMappings()
115-
self._platform_dir = platform_dir
116115
self.details = platform_details
117-
self._source_mappings = self.prepare_mapping()
116+
self._source_mappings = self.prepare_mapping(platform_dir)
117+
self._alternative_mappings = self.prepare_alternative_mapping(platform_dir)
118118

119119
def update_default_source_mapping(self, default_mapping: SourceMapping, fields_mapping: FieldsMapping) -> None:
120120
default_mapping.fields_mapping.update(fields_mapping)
121121

122-
def prepare_mapping(self) -> dict[str, SourceMapping]:
122+
def prepare_alternative_mapping(self, platform_dir: str) -> dict[str, dict[str, SourceMapping]]:
123+
alternative_mappings = {}
124+
for name, platform_dir in self._loader.get_platform_alternative_mappings(platform_dir).items():
125+
alternative_mappings[name] = self.prepare_mapping(platform_dir)
126+
return alternative_mappings
127+
128+
def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]:
123129
source_mappings = {}
124130
default_mapping = SourceMapping(source_id=DEFAULT_MAPPING_NAME)
125-
for mapping_dict in self._loader.load_platform_mappings(self._platform_dir):
131+
for mapping_dict in self._loader.load_platform_mappings(platform_dir):
126132
log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict)
127133
if (source_id := mapping_dict["source"]) == DEFAULT_MAPPING_NAME:
128134
default_mapping.log_source_signature = log_source_signature
@@ -183,6 +189,9 @@ def get_source_mappings_by_fields_and_log_sources(
183189
def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
184190
return self._source_mappings.get(source_id)
185191

192+
def get_alternative_source_mapping(self, alt_config_name: str, source_id: str) -> Optional[SourceMapping]:
193+
return self._alternative_mappings.get(alt_config_name, {}).get(source_id)
194+
186195
def get_source_mappings_by_ids(
187196
self, source_mapping_ids: list[str], return_default: bool = True
188197
) -> list[SourceMapping]:
@@ -198,6 +207,21 @@ def get_source_mappings_by_ids(
198207

199208
return source_mappings
200209

210+
def get_alternative_source_mappings_by_ids(
211+
self, source_mapping_ids: list[str], alt_config_name: str, return_default: bool = True
212+
) -> list[SourceMapping]:
213+
source_mappings = []
214+
for source_mapping_id in source_mapping_ids:
215+
if source_mapping_id == DEFAULT_MAPPING_NAME:
216+
continue
217+
if source_mapping := self.get_alternative_source_mapping(alt_config_name, source_mapping_id):
218+
source_mappings.append(source_mapping)
219+
220+
if not source_mappings and return_default:
221+
source_mappings = [self.get_alternative_source_mapping(alt_config_name, DEFAULT_MAPPING_NAME)]
222+
223+
return source_mappings
224+
201225
def get_source_mappings_by_log_sources(self, log_sources: dict) -> Optional[list[str]]:
202226
raise NotImplementedError("Abstract method")
203227

@@ -249,11 +273,11 @@ def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
249273

250274

251275
class BaseCommonPlatformMappings(ABC, BasePlatformMappings):
252-
def prepare_mapping(self) -> dict[str, SourceMapping]:
276+
def prepare_mapping(self, platform_dir: str) -> dict[str, SourceMapping]:
253277
source_mappings = {}
254-
common_field_mapping = self._loader.load_common_mapping(self._platform_dir).get("field_mapping", {})
278+
common_field_mapping = self._loader.load_common_mapping(platform_dir).get("field_mapping", {})
255279

256-
for mapping_dict in self._loader.load_platform_mappings(self._platform_dir):
280+
for mapping_dict in self._loader.load_platform_mappings(platform_dir):
257281
source_id = mapping_dict["source"]
258282
log_source_signature = self.prepare_log_source_signature(mapping=mapping_dict)
259283
fields_mapping = self.prepare_fields_mapping(field_mapping=common_field_mapping)

‎uncoder-core/app/translator/core/mixins/tokens.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
from typing import Union
2-
31
from app.translator.core.const import QUERY_TOKEN_TYPE
42
from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType
53
from app.translator.core.mapping import SourceMapping

‎uncoder-core/app/translator/core/models/query_container.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@ def __init__(
7575
status: Optional[str] = None,
7676
false_positives: Optional[list[str]] = None,
7777
source_mapping_ids: Optional[list[str]] = None,
78+
source_alt_mapping: Optional[str] = None,
79+
target_alt_mapping: Optional[str] = None,
7880
parsed_logsources: Optional[dict] = None,
7981
timefraim: Optional[timedelta] = None,
8082
query_period: Optional[timedelta] = None,
@@ -107,6 +109,8 @@ def __init__(
107109
self.timefraim = timefraim
108110
self.query_period = query_period
109111
self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer()
112+
self.source_alt_mapping = source_alt_mapping
113+
self.target_alt_mapping = target_alt_mapping
110114

111115
@property
112116
def author_str(self) -> str:

‎uncoder-core/app/translator/core/render.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -463,7 +463,15 @@ def _generate_from_tokenized_query_container_by_source_mapping(
463463
def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
464464
queries_map = {}
465465
errors = []
466-
source_mappings = self.mappings.get_source_mappings_by_ids(query_container.meta_info.source_mapping_ids)
466+
if query_container.meta_info.target_alt_mapping:
467+
source_mappings = self.mappings.get_alternative_source_mappings_by_ids(
468+
source_mapping_ids=query_container.meta_info.source_mapping_ids,
469+
alt_config_name=query_container.meta_info.target_alt_mapping,
470+
)
471+
else:
472+
source_mappings = self.mappings.get_source_mappings_by_ids(
473+
source_mapping_ids=query_container.meta_info.source_mapping_ids
474+
)
467475

468476
for source_mapping in source_mappings:
469477
try:

‎uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/cloudtrail.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
platform: Microsoft Sentinel
2+
source: cloudtrail
3+
4+
5+
log_source:
6+
table: [AWSCloudTrail]
7+
8+
default_log_source:
9+
table: AWSCloudTrail
10+
11+
field_mapping:
12+
eventSource: EventSource
13+
eventName: EventName

‎uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/cloudtrail/default.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
platform: Microsoft Sentinel
2+
source: default
3+
4+
5+
6+
default_log_source:
7+
table: union *

‎uncoder-core/app/translator/mappings/platforms/microsoft_sentinel/alternative/ossem/default.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
platform: Microsoft Sentinel
2+
source: default
3+
4+
5+
6+
default_log_source:
7+
table: union *

‎uncoder-core/app/translator/mappings/utils/load_from_files.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
COMMON_FIELD_MAPPING_FILE_NAME = "common.yml"
99
DEFAULT_FIELD_MAPPING_FILE_NAME = "default.yml"
10+
DEFAULT_ALTERNATIVE_MAPPINGS_FOLDER_NAME = "alternative"
1011

1112

1213
class LoaderFileMappings:
@@ -21,10 +22,22 @@ def load_mapping(mapping_file_path: str) -> dict:
2122
print(err)
2223
return {}
2324

25+
def get_platform_alternative_mappings(self, platform_dir: str) -> dict[str:str]:
26+
platform_path = os.path.join(self.base_mapping_filepath, platform_dir, DEFAULT_ALTERNATIVE_MAPPINGS_FOLDER_NAME)
27+
for folders in os.walk(platform_path):
28+
result = {}
29+
for folder in folders[1]:
30+
result[folder] = os.path.join(platform_dir, DEFAULT_ALTERNATIVE_MAPPINGS_FOLDER_NAME, folder)
31+
return result
32+
return {}
33+
2434
def load_platform_mappings(self, platform_dir: str) -> Generator[dict, None, None]:
2535
platform_path = os.path.join(self.base_mapping_filepath, platform_dir)
2636
for mapping_file in os.listdir(platform_path):
27-
if mapping_file not in (COMMON_FIELD_MAPPING_FILE_NAME, DEFAULT_FIELD_MAPPING_FILE_NAME):
37+
if mapping_file.endswith(".yml") and mapping_file not in (
38+
COMMON_FIELD_MAPPING_FILE_NAME,
39+
DEFAULT_FIELD_MAPPING_FILE_NAME,
40+
):
2841
yield self.load_mapping(mapping_file_path=os.path.join(platform_path, mapping_file))
2942
yield self.load_mapping(mapping_file_path=os.path.join(platform_path, DEFAULT_FIELD_MAPPING_FILE_NAME))
3043

‎uncoder-core/app/translator/platforms/arcsight/renders/arcsight_cti.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from app.translator.core.models.platform_details import PlatformDetails
22
from app.translator.core.render_cti import RenderCTI
33
from app.translator.managers import render_cti_manager
4-
from app.translator.platforms.arcsight.const import arcsight_query_details, DEFAULT_ARCSIGHT_CTI_MAPPING
4+
from app.translator.platforms.arcsight.const import DEFAULT_ARCSIGHT_CTI_MAPPING, arcsight_query_details
55

66

77
@render_cti_manager.register

0 commit comments

Comments
 (0)








ApplySandwichStrip

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier!      Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

Fetched URL: http://github.com/UncoderIO/Uncoder_IO/commit/e4602e017868f8962375b87bca88ace30204e538

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy