Skip to content

Commit 9a9f093

Browse files
tarnopolskyitarnopolskyi
committed
Merge branch 'prod' into 'gis-case-insensitive-sigma-mapping'
# Conflicts: # app/translator/platforms/palo_alto/renders/cortex_xsiam.py
1 parent 6c4dd6c commit 9a9f093

File tree

68 files changed

+278
-104
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+278
-104
lines changed

‎uncoder-core/app/translator/core/context_vars.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
from contextvars import ContextVar
22

33
return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False)
4-
"""Set to True to return ony first query if rendered multiple options"""
4+
"""Set to True to return only first query if rendered multiple options"""

‎uncoder-core/app/translator/core/custom_types/time.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
from app.translator.tools.custom_enum import CustomEnum
2+
3+
4+
class TimeFrameType(CustomEnum):
5+
years = "years"
6+
months = "months"
7+
days = "days"
8+
hours = "hours"
9+
minutes = "minutes"
10+
11+
12+
class TimePartType(CustomEnum):
13+
day = "day"
14+
day_of_week = "day_of_week"
15+
day_of_year = "day_of_year"
16+
hour = "hour"
17+
microsecond = "microsecond"
18+
millisecond = "millisecond"
19+
minute = "minute"
20+
month = "month"
21+
quarter = "quarter"
22+
second = "second"
23+
year = "year"

‎uncoder-core/app/translator/core/exceptions/core.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
from typing import Optional
2+
3+
14
class NotImplementedException(BaseException):
25
...
36

@@ -7,8 +10,19 @@ class BasePlatformException(BaseException):
710

811

912
class StrictPlatformException(BasePlatformException):
10-
def __init__(self, platform_name: str, field_name: str):
11-
message = f"Platform {platform_name} has strict mapping. Source field {field_name} has no mapping."
13+
field_name: str = None
14+
15+
def __init__(
16+
self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None
17+
):
18+
message = (
19+
f"Platform {platform_name} has strict mapping. "
20+
f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping."
21+
f" Mapping file: {mapping}."
22+
if mapping
23+
else ""
24+
)
25+
self.field_name = field_name
1226
super().__init__(message)
1327

1428

‎uncoder-core/app/translator/core/mapping.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ def is_suitable(self, *args, **kwargs) -> bool:
2020
def __str__(self) -> str:
2121
raise NotImplementedError("Abstract method")
2222

23+
@property
24+
def default_source(self) -> dict:
25+
return self._default_source
26+
2327

2428
class FieldMapping:
2529
def __init__(self, generic_field_name: str, platform_field_name: str):

‎uncoder-core/app/translator/core/parser.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1616
-----------------------------------------------------------------
1717
"""
18+
1819
import re
1920
from abc import ABC, abstractmethod
2021
from typing import Union

‎uncoder-core/app/translator/core/render.py

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
1920
from abc import ABC, abstractmethod
2021
from collections.abc import Callable
2122
from typing import ClassVar, Optional, Union
@@ -196,10 +197,10 @@ class PlatformQueryRender(QueryRender):
196197
not_token = "not"
197198

198199
group_token = "(%s)"
200+
query_parts_delimiter = " "
199201

200202
field_value_map = BaseQueryFieldValue(or_token=or_token)
201203

202-
query_pattern = "{table} {query} {functions}"
203204
raw_log_field_pattern_map: ClassVar[dict[str, str]] = None
204205

205206
def __init__(self):
@@ -210,9 +211,9 @@ def __init__(self):
210211
LogicalOperatorType.NOT: f" {self.not_token} ",
211212
}
212213

213-
def generate_prefix(self, log_source_signature: LogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002
214-
if str(log_source_signature):
215-
return f"{log_source_signature!s} {self.and_token}"
214+
def generate_prefix(self, log_source_signature: Optional[LogSourceSignature], functions_prefix: str = "") -> str: # noqa: ARG002
215+
if log_source_signature and str(log_source_signature):
216+
return f"{log_source_signature} {self.and_token}"
216217
return ""
217218

218219
def generate_functions(self, functions: list[Function], source_mapping: SourceMapping) -> RenderedFunctions:
@@ -262,8 +263,14 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp
262263

263264
def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str:
264265
result_values = []
266+
unmapped_fields = set()
265267
for token in tokens:
266-
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
268+
try:
269+
result_values.append(self.apply_token(token=token, source_mapping=source_mapping))
270+
except StrictPlatformException as err:
271+
unmapped_fields.add(err.field_name)
272+
if unmapped_fields:
273+
raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields))
267274
return "".join(result_values)
268275

269276
def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str:
@@ -280,6 +287,14 @@ def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) ->
280287
query = f"{query}\n\n{query_meta_info}"
281288
return query
282289

290+
@staticmethod
291+
def _finalize_search_query(query: str) -> str:
292+
return query
293+
294+
def _join_query_parts(self, prefix: str, query: str, functions: str) -> str:
295+
parts = filter(lambda s: bool(s), map(str.strip, [prefix, self._finalize_search_query(query), functions]))
296+
return self.query_parts_delimiter.join(parts)
297+
283298
def finalize_query(
284299
self,
285300
prefix: str,
@@ -291,8 +306,7 @@ def finalize_query(
291306
*args, # noqa: ARG002
292307
**kwargs, # noqa: ARG002
293308
) -> str:
294-
query = self.query_pattern.format(prefix=prefix, query=query, functions=functions).strip()
295-
309+
query = self._join_query_parts(prefix, query, functions)
296310
query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query)
297311
if not_supported_functions:
298312
rendered_not_supported = self.render_not_supported_functions(not_supported_functions)
@@ -335,15 +349,15 @@ def _generate_from_raw_query_container(self, query_container: RawQueryContainer)
335349

336350
def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]:
337351
if raw_log_field_pattern := self.raw_log_field_pattern_map.get(field_type):
338-
return raw_log_field_pattern.pattern.format(field=field)
352+
return raw_log_field_pattern.format(field=field)
339353

340354
def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping) -> Optional[list]:
341355
if isinstance(field, list):
342-
list_of_prefix = []
356+
prefix_list = []
343357
for f in field:
344-
if prepared_prefix := self.process_raw_log_field_prefix(field=f, source_mapping=source_mapping):
345-
list_of_prefix.extend(prepared_prefix)
346-
return list_of_prefix
358+
if _prefix_list := self.process_raw_log_field_prefix(field=f, source_mapping=source_mapping):
359+
prefix_list.extend(_prefix_list)
360+
return prefix_list
347361
if raw_log_field_type := source_mapping.raw_log_fields.get(field):
348362
return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)]
349363

@@ -360,9 +374,11 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap
360374
)
361375
if not mapped_field and self.is_strict_mapping:
362376
raise StrictPlatformException(field_name=field.source_name, platform_name=self.details.name)
363-
if field_prefix := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
364-
defined_raw_log_fields.extend(field_prefix)
365-
return "\n".join(set(defined_raw_log_fields))
377+
if prefix_list := self.process_raw_log_field_prefix(field=mapped_field, source_mapping=source_mapping):
378+
for prefix in prefix_list:
379+
if prefix not in defined_raw_log_fields:
380+
defined_raw_log_fields.append(prefix)
381+
return "\n".join(defined_raw_log_fields)
366382

367383
def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str:
368384
queries_map = {}
@@ -377,7 +393,7 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue
377393
defined_raw_log_fields = self.generate_raw_log_fields(
378394
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
379395
)
380-
prefix += f"\n{defined_raw_log_fields}\n"
396+
prefix += f"\n{defined_raw_log_fields}"
381397
result = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
382398
except StrictPlatformException as err:
383399
errors.append(err)

‎uncoder-core/app/translator/core/render_cti.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
-----------------------------------------------------------------
1818
"""
1919

20-
2120
from app.translator.core.models.iocs import IocsChunkValue
2221
from app.translator.core.models.platform_details import PlatformDetails
2322

‎uncoder-core/app/translator/core/str_value_manager.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
limitations under the License.
1717
-----------------------------------------------------------------
1818
"""
19+
1920
from typing import ClassVar, Optional, TypeVar, Union
2021

2122
from app.translator.core.custom_types.values import ValueType

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/aws_cloudtrail.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,5 @@ raw_log_fields:
3232
userIdentity.principalId: object
3333
userIdentity.sessionContext.sessionIssuer.type: object
3434
userIdentity.type: object
35-
userIdentity.userName: object
35+
userIdentity.userName: object
36+
requestParameters.publiclyAccessible: object

‎uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/azure_signinlogs.yml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
platform: Palo Alto XSIAM
2+
source: azure_signinlogs
3+
4+
5+
default_log_source:
6+
dataset: msft_azure_raw
7+
8+
field_mapping:
9+
AppDisplayName: properties.appDisplayName
10+
AppId: properties.appId
11+
AuthenticationRequirement: properties.authenticationRequirement
12+
Category: properties.category
13+
ConditionalAccessStatus: properties.conditionalAccessStatus
14+
DeviceDetail: properties.deviceDetail
15+
IsInteractive: properties.isInteractive
16+
NetworkLocationDetails: properties.networkLocationDetails
17+
ResourceDisplayName: properties.resourceDisplayName
18+
ResourceIdentity: properties.resourceIdentity
19+
ResultDescription: properties.resultDescription
20+
ResultType: properties.resultType
21+
Status.errorCode: properties.status.errorCode
22+
Status: properties.status
23+
Status.failureReason: properties.status.failureReason
24+
TokenIssuerType: properties.tokenIssuerType
25+
UserAgent: properties.userAgent
26+
UserPrincipalName: properties.userPrincipalName
27+
28+
raw_log_fields:
29+
properties.appDisplayName: object
30+
properties.appId: object
31+
properties.authenticationRequirement: object
32+
properties.category: object
33+
properties.conditionalAccessStatus: object
34+
properties.deviceDetail: object
35+
properties.isInteractive: object
36+
properties.networkLocationDetails: object
37+
properties.resourceDisplayName: object
38+
properties.resourceIdentity: object
39+
properties.resultDescription: object
40+
properties.resultType: object
41+
properties.status.errorCode: object
42+
properties.status: object
43+
properties.status.failureReason: object
44+
properties.tokenIssuerType: object
45+
properties.userAgent: object
46+
properties.userPrincipalName: object

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy