Skip to content

Commit ee94d17

Browse files
a-alferova-alferov
committed
ADCM-5621 Implement encoder, fix controller
1 parent bffebbb commit ee94d17

File tree

9 files changed

+215
-65
lines changed

9 files changed

+215
-65
lines changed

‎data/tmp/.gitkeep

Whitespace-only changes.

‎python/adcm/settings.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
FILE_DIR = STACK_DIR / "data" / "file"
3838
LOG_DIR = DATA_DIR / "log"
3939
VAR_DIR = DATA_DIR / "var"
40+
TMP_DIR = DATA_DIR / "tmp"
4041
LOG_FILE = LOG_DIR / "adcm.log"
4142
SECRETS_FILE = VAR_DIR / "secrets.json"
4243
ADCM_TOKEN_FILE = VAR_DIR / "adcm_token"

‎python/adcm/tests/base.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ def _prepare_temporal_directories_for_adcm() -> dict:
104104
"FILE_DIR": stack / "data" / "file",
105105
"LOG_DIR": data / "log",
106106
"VAR_DIR": data / "var",
107+
"TMP_DIR": data / "tmp",
107108
}
108109

109110
for directory in temporary_directories.values():

‎python/audit/utils.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
from contextlib import suppress
1414
from functools import wraps
15+
from typing import Callable
1516
import re
1617

1718
from api.cluster.serializers import ClusterAuditSerializer
@@ -648,3 +649,35 @@ def audit_job_finish(owner: NamedCoreObject, display_name: str, is_upgrade: bool
648649
)
649650

650651
cef_logger(audit_instance=audit_log, signature_id="Action completion")
652+
653+
654+
def audit_background_task(start_operation_status: str, end_operation_status: str) -> Callable:
655+
def decorator(func: Callable) -> Callable:
656+
@wraps(func)
657+
def wrapped(*args, **kwargs):
658+
make_audit_log(
659+
operation_type="statistics",
660+
result=AuditLogOperationResult.SUCCESS,
661+
operation_status=start_operation_status,
662+
)
663+
try:
664+
result = func(*args, **kwargs)
665+
except Exception as error:
666+
make_audit_log(
667+
operation_type="statistics",
668+
result=AuditLogOperationResult.FAIL,
669+
operation_status=end_operation_status,
670+
)
671+
raise error
672+
673+
make_audit_log(
674+
operation_type="statistics",
675+
result=AuditLogOperationResult.SUCCESS,
676+
operation_status=end_operation_status,
677+
)
678+
679+
return result
680+
681+
return wrapped
682+
683+
return decorator

‎python/cm/collect_statistics/collectors.py

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from hashlib import md5
1515
from typing import Collection, Literal
1616

17-
from django.db.models import Count, F
17+
from django.db.models import Count, F, Q
1818
from pydantic import BaseModel
1919
from rbac.models import Policy, Role, User
2020
from typing_extensions import TypedDict
@@ -89,16 +89,14 @@ def __call__(self) -> RBACEntities:
8989

9090

9191
class BundleCollector:
92-
def __init__(self, date_format: str, include_editions: Collection[str]):
92+
def __init__(self, date_format: str, filters: Collection[Q]):
9393
self._date_format = date_format
94-
self._editions = include_editions
94+
self._filters = filters
9595

9696
def __call__(self) -> ADCMEntities:
9797
bundles: dict[int, BundleData] = {
9898
entry.pop("id"): BundleData(date=entry.pop("date").strftime(self._date_format), **entry)
99-
for entry in Bundle.objects.filter(edition__in=self._editions).values(
100-
"id", *BundleData.__annotations__.keys()
101-
)
99+
for entry in Bundle.objects.filter(*self._filters).values("id", *BundleData.__annotations__.keys())
102100
}
103101

104102
hostproviders_data = [

‎python/cm/collect_statistics/encoders.py

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,27 @@
1616

1717

1818
class TarFileEncoder(Encoder[Path]):
19-
def encode(self, data: Path) -> Path:
20-
pass
19+
"""Encode and decode a file in place"""
2120

22-
def decode(self, data: Path) -> Path:
23-
pass
21+
__slots__ = ["suffix"]
22+
23+
def __init__(self, suffix: str) -> None:
24+
if suffix and not suffix.startswith(".") or suffix == ".":
25+
raise ValueError(f"Invalid suffix '{suffix}'")
26+
27+
self.suffix = suffix
28+
29+
def encode(self, path_file: Path) -> Path:
30+
encoded_data = bytearray((byte + 1) % 256 for byte in path_file.read_bytes())
31+
encoded_file = path_file.rename(path_file.parent / f"{path_file.name}{self.suffix}")
32+
encoded_file.write_bytes(encoded_data)
33+
return encoded_file
34+
35+
def decode(self, path_file: Path) -> Path:
36+
if not path_file.name.endswith(self.suffix):
37+
raise ValueError(f"The file name must end with '{self.suffix}'")
38+
39+
decoded_data = bytearray((byte - 1) % 256 for byte in path_file.read_bytes())
40+
decoded_file = path_file.rename(path_file.parent / path_file.name[: -len(self.suffix)])
41+
decoded_file.write_bytes(decoded_data)
42+
return decoded_file

‎python/cm/collect_statistics/storages.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,13 @@ class JSONFile(BaseModel):
2929

3030

3131
class TarFileWithJSONFileStorage(Storage[JSONFile]):
32-
def __init__(self, compresslevel=9, timeformat="%Y-%m-%d"):
32+
__slots__ = ("json_files", "tmp_dir", "compresslevel", "date_format")
33+
34+
def __init__(self, compresslevel=9, date_format="%Y-%m-%d"):
3335
self.json_files = []
3436
self.tmp_dir = Path(mkdtemp()).absolute()
3537
self.compresslevel = compresslevel
36-
self.timeformat = timeformat
38+
self.date_format = date_format
3739

3840
def add(self, data: JSONFile) -> None:
3941
"""
@@ -66,8 +68,8 @@ def gather(self) -> Path:
6668
if not self:
6769
raise StorageError("No JSON files to gather")
6870

69-
today_date = datetime.datetime.now(tz=datetime.timezone.utc).strftime(self.timeformat)
70-
archive_name = self.tmp_dir / f"{today_date}_statistics_full.tgz"
71+
today_date = datetime.datetime.now(tz=datetime.timezone.utc).strftime(self.date_format)
72+
archive_name = self.tmp_dir / f"{today_date}_statistics.tar.gz"
7173
archive_path = Path(archive_name)
7274

7375
with tarfile.open(archive_name, "w:gz", compresslevel=self.compresslevel) as tar:

‎python/cm/management/commands/collect_statistics_new.py

Lines changed: 88 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -10,26 +10,35 @@
1010
# See the License for the specific language governing permissions and
1111
# limitations under the License.
1212

13+
from logging import getLogger
1314
from typing import NamedTuple
1415
from urllib.parse import urlunparse
1516
import os
1617
import socket
1718

19+
from audit.utils import audit_background_task
1820
from django.conf import settings
1921
from django.core.management import BaseCommand
22+
from django.db.models import Q
23+
from django.utils import timezone
2024

2125
from cm.adcm_config.config import get_adcm_config
2226
from cm.collect_statistics.collectors import ADCMEntities, BundleCollector, RBACCollector
2327
from cm.collect_statistics.encoders import TarFileEncoder
2428
from cm.collect_statistics.senders import SenderSettings, StatisticSender
25-
from cm.collect_statistics.storages import JSONFile, TarFileWithJSONFileStorage, TarFileWithTarFileStorage
29+
from cm.collect_statistics.storages import JSONFile, TarFileWithJSONFileStorage
2630
from cm.models import ADCM
2731

2832
SENDER_REQUEST_TIMEOUT = 15.0
29-
DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
33+
DATE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
34+
DATE_FORMAT = "%Y-%m-%d"
35+
STATISTIC_DIR = settings.TMP_DIR / "statistics"
36+
STATISTIC_DIR.mkdir(exist_ok=True)
3037

31-
collect_community = BundleCollector(date_format=DATE_FORMAT, include_editions=["community"])
32-
collect_enterprise = BundleCollector(date_format=DATE_FORMAT, include_editions=["enterprise"])
38+
logger = getLogger("background_tasks")
39+
40+
collect_not_enterprise = BundleCollector(date_format=DATE_TIME_FORMAT, filters=[~Q(edition="enterprise")])
41+
collect_all = BundleCollector(date_format=DATE_TIME_FORMAT, filters=[])
3342

3443

3544
class URLComponents(NamedTuple):
@@ -64,18 +73,34 @@ def get_statistics_url() -> str:
6473
return urlunparse(components=URLComponents(scheme=scheme, netloc=netloc, path=url_path))
6574

6675

76+
def get_enabled() -> bool:
77+
if os.getenv("STATISTICS_ENABLED") is not None:
78+
return os.environ["STATISTICS_ENABLED"].upper() in {"1", "TRUE"}
79+
80+
attr, _ = get_adcm_config(section="statistics_collection")
81+
return bool(attr["active"])
82+
83+
6784
class Command(BaseCommand):
6885
help = "Collect data and send to Statistic Server"
6986

7087
def __init__(self, *args, **kwargs):
7188
super().__init__(*args, **kwargs)
7289

7390
def add_arguments(self, parser):
74-
parser.add_argument("--full", action="store_true", help="collect all data")
75-
parser.add_argument("--send", action="store_true", help="send data to Statistic Server")
76-
parser.add_argument("--encode", action="store_true", help="encode data")
91+
parser.add_argument(
92+
"--mode",
93+
choices=["send", "archive-all"],
94+
help=(
95+
"'send' - collect archive with only community bundles and send to Statistic Server, "
96+
"'archive-all' - collect community and enterprise bundles to archive and return path to file"
97+
),
98+
default="archive-all",
99+
)
77100

78-
def handle(self, *_, full: bool, send: bool, encode: bool, **__):
101+
@audit_background_task(start_operation_status="launched", end_operation_status="completed")
102+
def handle(self, *_, mode: str, **__):
103+
logger.debug(msg="Statistics collector: started")
79104
statistics_data = {
80105
"adcm": {
81106
"uuid": str(ADCM.objects.values_list("uuid", flat=True).get()),
@@ -84,47 +109,63 @@ def handle(self, *_, full: bool, send: bool, encode: bool, **__):
84109
},
85110
"format_version": "0.2",
86111
}
87-
rbac_entries_data: dict = RBACCollector(date_format=DATE_FORMAT)().model_dump()
112+
logger.debug(msg="Statistics collector: RBAC data preparation")
113+
rbac_entries_data: dict = RBACCollector(date_format=DATE_TIME_FORMAT)().model_dump()
114+
storage = TarFileWithJSONFileStorage(date_format=DATE_FORMAT)
88115

89-
community_bundle_data: ADCMEntities = collect_community()
90-
community_storage = TarFileWithJSONFileStorage()
116+
match mode:
117+
case "send":
118+
logger.debug(msg="Statistics collector: 'send' mode is used")
91119

92-
community_storage.add(
93-
JSONFile(
94-
filename="community.json",
95-
data={**statistics_data, **rbac_entries_data, **community_bundle_data.model_dump()},
96-
)
97-
)
98-
community_archive = community_storage.gather()
120+
if not get_enabled():
121+
logger.debug(msg="Statistics collector: disabled")
122+
return
99123

100-
final_storage = TarFileWithTarFileStorage()
101-
final_storage.add(community_archive)
124+
logger.debug(
125+
msg="Statistics collector: bundles data preparation, collect everything except 'enterprise' edition"
126+
)
127+
bundle_data: ADCMEntities = collect_not_enterprise()
128+
storage.add(
129+
JSONFile(
130+
filename=f"{timezone.now().strftime(DATE_FORMAT)}_statistics.json",
131+
data={**statistics_data, **rbac_entries_data, **bundle_data.model_dump()},
132+
)
133+
)
134+
logger.debug(msg="Statistics collector: archive preparation")
135+
archive = storage.gather()
136+
sender_settings = SenderSettings(
137+
url=get_statistics_url(),
138+
adcm_uuid=statistics_data["adcm"]["uuid"],
139+
retries_limit=int(os.getenv("STATISTICS_RETRIES", 10)),
140+
retries_frequency=int(os.getenv("STATISTICS_FREQUENCY", 1 * 60 * 60)), # in seconds
141+
request_timeout=SENDER_REQUEST_TIMEOUT,
142+
)
143+
logger.debug(msg="Statistics collector: sender preparation")
144+
sender = StatisticSender(settings=sender_settings)
145+
logger.debug(msg="Statistics collector: statistics sending has started")
146+
sender.send([archive])
147+
logger.debug(msg="Statistics collector: sending statistics completed")
148+
149+
case "archive-all":
150+
logger.debug(msg="Statistics collector: 'archive-all' mode is used")
151+
logger.debug(msg="Statistics collector: bundles data preparation, collect everything")
152+
bundle_data: ADCMEntities = collect_all()
153+
storage.add(
154+
JSONFile(
155+
filename=f"{timezone.now().strftime(DATE_FORMAT)}_statistics.json",
156+
data={**statistics_data, **rbac_entries_data, **bundle_data.model_dump()},
157+
)
158+
)
159+
logger.debug(msg="Statistics collector: archive preparation")
160+
archive = storage.gather()
102161

103-
if full:
104-
enterprise_bundle_data: ADCMEntities = collect_enterprise()
105-
enterprise_storage = TarFileWithJSONFileStorage()
162+
logger.debug(msg="Statistics collector: archive encoding")
163+
encoder = TarFileEncoder(suffix=".enc")
164+
encoded_file = encoder.encode(path_file=archive)
165+
encoded_file = encoded_file.replace(STATISTIC_DIR / encoded_file.name)
106166

107-
enterprise_storage.add(
108-
JSONFile(
109-
filename="enterprise.json",
110-
data={**statistics_data, **rbac_entries_data, **enterprise_bundle_data.model_dump()},
111-
)
112-
)
113-
final_storage.add(enterprise_storage.gather())
114-
115-
final_archive = final_storage.gather()
116-
117-
if encode:
118-
encoder = TarFileEncoder()
119-
encoder.encode(final_archive)
120-
121-
if send:
122-
sender_settings = SenderSettings(
123-
url=get_statistics_url(),
124-
adcm_uuid=statistics_data["adcm"]["uuid"],
125-
retries_limit=int(os.getenv("STATISTICS_RETRIES", 10)),
126-
retries_frequency=int(os.getenv("STATISTICS_FREQUENCY", 1 * 60 * 60)), # in seconds
127-
request_timeout=SENDER_REQUEST_TIMEOUT,
128-
)
129-
sender = StatisticSender(settings=sender_settings)
130-
sender.send([community_archive])
167+
self.stdout.write(f"Data saved in: {str(encoded_file.absolute())}")
168+
case _:
169+
pass
170+
171+
logger.debug(msg="Statistics collector: finished")

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy