From 4f9807f2fc137b81ef79c6f7d9a7d67c16f73a17 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Wed, 30 Mar 2022 23:04:20 +0200 Subject: [PATCH 01/12] feat(downloads): allow streaming downloads access to response iterator Allow access to the underlying response iterator when downloading in streaming mode by specifying action="iterator" Update type annotations to support this change --- gitlab/mixins.py | 6 ++++-- gitlab/utils.py | 9 ++++++--- gitlab/v4/cli.py | 3 ++- gitlab/v4/objects/artifacts.py | 15 ++++++++------- gitlab/v4/objects/files.py | 17 ++++++++++++++--- gitlab/v4/objects/jobs.py | 20 +++++++++++++++----- gitlab/v4/objects/packages.py | 15 ++++++++++++--- gitlab/v4/objects/projects.py | 19 +++++++++++++++---- gitlab/v4/objects/repositories.py | 20 +++++++++++++++----- gitlab/v4/objects/snippets.py | 20 +++++++++++++++----- 10 files changed, 106 insertions(+), 38 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index 1a3ff4dbf..f8b5a8a30 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -20,7 +20,9 @@ Any, Callable, Dict, + Iterator, List, + Literal, Optional, Tuple, Type, @@ -657,10 +659,10 @@ class DownloadMixin(_RestObjectBase): def download( self, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download the archive of a resource export. Args: diff --git a/gitlab/utils.py b/gitlab/utils.py index 197935549..0193a7562 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -19,7 +19,7 @@ import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Optional, Type, Union +from typing import Any, Callable, Dict, Iterator, Literal, Optional, Type, Union import requests @@ -32,15 +32,18 @@ def __call__(self, chunk: Any) -> None: def response_content( response: requests.Response, streamed: bool, - action: Optional[Callable], + action: Optional[Union[Callable, Literal["iterator"]]], chunk_size: int, -) -> Optional[bytes]: +) -> Optional[Union[bytes, Iterator[Any]]]: if streamed is False: return response.content if action is None: action = _StdoutStream() + if action == "iterator": + return response.iter_content(chunk_size=chunk_size) + for chunk in response.iter_content(chunk_size=chunk_size): if chunk: action(chunk) diff --git a/gitlab/v4/cli.py b/gitlab/v4/cli.py index 6830b0874..f9349bdbd 100644 --- a/gitlab/v4/cli.py +++ b/gitlab/v4/cli.py @@ -19,7 +19,7 @@ import argparse import operator import sys -from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING, Union +from typing import Any, Dict, Iterator, List, Optional, Type, TYPE_CHECKING, Union import gitlab import gitlab.base @@ -123,6 +123,7 @@ def do_project_export_download(self) -> None: data = export_status.download() if TYPE_CHECKING: assert data is not None + assert not isinstance(data, Iterator) sys.stdout.buffer.write(data) except Exception as e: diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index 541e5e2f4..cdcfda7c1 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,7 +2,7 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -from typing import Any, Callable, Optional, TYPE_CHECKING +from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union import requests @@ -32,7 +32,7 @@ def __call__( self, *args: Any, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: utils.warn( message=( "The project.artifacts() method is deprecated and will be removed in a " @@ -71,10 +71,10 @@ def download( ref_name: str, job: str, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get the job artifacts archive from a specific tag or branch. Args: @@ -86,7 +86,8 @@ def download( `chunk_size` and each chunk is passed to `action` for treatment action: Callable responsible of dealing with chunk of - data + data. May also be the string "iterator" to directly return + the response iterator chunk_size: Size of each chunk **kwargs: Extra options to send to the server (e.g. sudo) @@ -115,10 +116,10 @@ def raw( artifact_path: str, job: str, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download a single artifact file from a specific tag or branch from within the job's artifacts archive. diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index 435e71b55..1f4d98ccf 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,5 +1,16 @@ import base64 -from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -217,10 +228,10 @@ def raw( file_path: str, ref: str, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a file for a commit. Args: diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index fbcb1fd40..db278287a 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,4 +1,14 @@ -from typing import Any, Callable, cast, Dict, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -116,10 +126,10 @@ def delete_artifacts(self, **kwargs: Any) -> None: def artifacts( self, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get the job artifacts. Args: @@ -152,10 +162,10 @@ def artifact( self, path: str, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get a single artifact file from within the job's artifacts archive. Args: diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 0461bdcd9..1f8539558 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -5,7 +5,16 @@ """ from pathlib import Path -from typing import Any, Callable, cast, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Iterator, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -103,10 +112,10 @@ def download( package_version: str, file_name: str, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download a generic package. Args: diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index 81eb62496..cd4aafd6c 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,4 +1,15 @@ -from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -457,10 +468,10 @@ def snapshot( self, wiki: bool = False, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return a snapshot of the repository. Args: @@ -562,7 +573,7 @@ def artifact( self, *args: Any, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: utils.warn( message=( "The project.artifact() method is deprecated and will be " diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index f2792b14e..9ea5e5237 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,7 +3,17 @@ Currently this module only contains repository-related methods for projects. """ -from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -107,10 +117,10 @@ def repository_raw_blob( self, sha: str, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the raw file contents for a blob. Args: @@ -192,11 +202,11 @@ def repository_archive( self, sha: str = None, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, format: Optional[str] = None, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return an archive of the repository. Args: diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 9d9dcc4e6..6d06fc7ed 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,4 +1,14 @@ -from typing import Any, Callable, cast, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -28,10 +38,10 @@ class Snippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObject): def content( self, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a snippet. Args: @@ -102,10 +112,10 @@ class ProjectSnippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObj def content( self, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a snippet. Args: From efd8b486605bb38a5b0e8f5066eed675ed389836 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 9 Apr 2022 15:37:47 +0200 Subject: [PATCH 02/12] feat(downloads): add conditional dependency on literal for python 3.7 This supports commit c76b3b14b65a19b4d6b2a745745b3e493efa8c68 and makes sure it works on python 3.7 as well. --- gitlab/mixins.py | 7 ++++++- gitlab/utils.py | 8 +++++++- gitlab/v4/objects/artifacts.py | 8 +++++++- gitlab/v4/objects/files.py | 7 ++++++- gitlab/v4/objects/jobs.py | 18 +++++++----------- gitlab/v4/objects/packages.py | 17 +++++++---------- gitlab/v4/objects/projects.py | 7 ++++++- gitlab/v4/objects/repositories.py | 18 +++++++----------- gitlab/v4/objects/snippets.py | 18 +++++++----------- 9 files changed, 60 insertions(+), 48 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index f8b5a8a30..68388a410 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +import sys from types import ModuleType from typing import ( Any, @@ -22,7 +23,6 @@ Dict, Iterator, List, - Literal, Optional, Tuple, Type, @@ -38,6 +38,11 @@ from gitlab import types as g_types from gitlab import utils +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "GetMixin", "GetWithoutIdMixin", diff --git a/gitlab/utils.py b/gitlab/utils.py index 0193a7562..46b4ea773 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -16,13 +16,19 @@ # along with this program. If not, see . import pathlib +import sys import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Iterator, Literal, Optional, Type, Union +from typing import Any, Callable, Dict, Iterator, Optional, Type, Union import requests +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + class _StdoutStream: def __call__(self, chunk: Any) -> None: diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index cdcfda7c1..d78efae21 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,7 +2,8 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union +import sys +from typing import Any, Callable, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -11,6 +12,11 @@ from gitlab import utils from gitlab.base import RESTManager, RESTObject +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = ["ProjectArtifact", "ProjectArtifactManager"] diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index 1f4d98ccf..3476a18c0 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,4 +1,5 @@ import base64 +import sys from typing import ( Any, Callable, @@ -6,7 +7,6 @@ Dict, Iterator, List, - Literal, Optional, TYPE_CHECKING, Union, @@ -27,6 +27,11 @@ UpdateMixin, ) +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "ProjectFile", "ProjectFileManager", diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index db278287a..ef018ca3f 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,14 +1,5 @@ -from typing import ( - Any, - Callable, - cast, - Dict, - Iterator, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +import sys +from typing import Any, Callable, cast, Dict, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -18,6 +9,11 @@ from gitlab.base import RESTManager, RESTObject from gitlab.mixins import RefreshMixin, RetrieveMixin +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "ProjectJob", "ProjectJobManager", diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 1f8539558..60a06d8d3 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -4,17 +4,9 @@ https://docs.gitlab.com/ee/user/packages/generic_packages/ """ +import sys from pathlib import Path -from typing import ( - Any, - Callable, - cast, - Iterator, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -24,6 +16,11 @@ from gitlab.base import RESTManager, RESTObject from gitlab.mixins import DeleteMixin, GetMixin, ListMixin, ObjectDeleteMixin +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "GenericPackage", "GenericPackageManager", diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index cd4aafd6c..74053c13e 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,3 +1,4 @@ +import sys from typing import ( Any, Callable, @@ -5,7 +6,6 @@ Dict, Iterator, List, - Literal, Optional, TYPE_CHECKING, Union, @@ -82,6 +82,11 @@ from .variables import ProjectVariableManager # noqa: F401 from .wikis import ProjectWikiManager # noqa: F401 +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "GroupProject", "GroupProjectManager", diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index 9ea5e5237..b8af26979 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,17 +3,8 @@ Currently this module only contains repository-related methods for projects. """ -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +import sys +from typing import Any, Callable, Dict, Iterator, List, Optional, TYPE_CHECKING, Union import requests @@ -22,6 +13,11 @@ from gitlab import exceptions as exc from gitlab import utils +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + if TYPE_CHECKING: # When running mypy we use these as the base classes _RestObjectBase = gitlab.base.RESTObject diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 6d06fc7ed..8c4dd7159 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,14 +1,5 @@ -from typing import ( - Any, - Callable, - cast, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +import sys +from typing import Any, Callable, cast, Iterator, List, Optional, TYPE_CHECKING, Union import requests @@ -22,6 +13,11 @@ from .discussions import ProjectSnippetDiscussionManager # noqa: F401 from .notes import ProjectSnippetNoteManager # noqa: F401 +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "Snippet", "SnippetManager", From 8cc4cfc0e78b8e1f430ed9e23089423cad3381ed Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 9 Apr 2022 15:41:35 +0200 Subject: [PATCH 03/12] chore(deps): add conditional dependency on typing_extensions This is used to support Literal from typing on python < 3.8 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index c94a1d220..cccf30dd7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests==2.27.1 requests-toolbelt==0.9.1 +typing_extensions; python_version<"3.8" \ No newline at end of file From 63fa05b8eb29ccd732b118b144e4812041a94b03 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 10 Apr 2022 10:16:11 +0200 Subject: [PATCH 04/12] docs(api-docs): add iterator example to artifact download Document the usage of the action="iterator" option when downloading artifacts --- docs/gl_objects/pipelines_and_jobs.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/gl_objects/pipelines_and_jobs.rst b/docs/gl_objects/pipelines_and_jobs.rst index a05d968a4..78cae51ba 100644 --- a/docs/gl_objects/pipelines_and_jobs.rst +++ b/docs/gl_objects/pipelines_and_jobs.rst @@ -274,6 +274,18 @@ You can also directly stream the output into a file, and unzip it afterwards:: subprocess.run(["unzip", "-bo", zipfn]) os.unlink(zipfn) +It is also possible to use the underlying iterator directly:: + + artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') + +This can be used with FastAPI/Starlette StreamingResponse to forward a download from gitlab without having to download +the entire file server side first:: + + @app.get("/download_artifact") + def download_artifact(): + artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') + return StreamingResponse(artifact_bytes_iterator, media_type="application/zip") + Delete all artifacts of a project that can be deleted:: project.artifacts.delete() From d7ee6f8e556b7829e41b3fa800cf07addfe9a564 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 10 Apr 2022 10:57:35 +0200 Subject: [PATCH 05/12] test(packages): add tests for streaming downloads Tests for the functionality implemented in 4f9807f2fc137b81ef79c6f7d9a7d67c16f73a17 --- tests/functional/api/test_packages.py | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/functional/api/test_packages.py b/tests/functional/api/test_packages.py index 64b57b827..e009572c4 100644 --- a/tests/functional/api/test_packages.py +++ b/tests/functional/api/test_packages.py @@ -3,6 +3,8 @@ https://docs.gitlab.com/ce/api/packages.html https://docs.gitlab.com/ee/user/packages/generic_packages """ +from collections.abc import Iterator + from gitlab.v4.objects import GenericPackage package_name = "hello-world" @@ -46,6 +48,25 @@ def test_download_generic_package(project): assert package.decode("utf-8") == file_content +def test_stream_generic_package(project): + bytes_iterator = project.generic_packages.download( + package_name=package_name, + package_version=package_version, + file_name=file_name, + streamed=True, + action="iterator", + ) + + assert isinstance(bytes_iterator, Iterator) + + package = bytes() + for chunk in bytes_iterator: + package += chunk + + assert isinstance(package, bytes) + assert package.decode("utf-8") == file_content + + def test_download_generic_package_to_file(tmp_path, project): path = tmp_path / file_name @@ -60,3 +81,22 @@ def test_download_generic_package_to_file(tmp_path, project): with open(path, "r") as f: assert f.read() == file_content + + +def test_stream_generic_package_to_file(tmp_path, project): + path = tmp_path / file_name + + bytes_iterator = project.generic_packages.download( + package_name=package_name, + package_version=package_version, + file_name=file_name, + streamed=True, + action="iterator", + ) + + with open(path, "wb") as f: + for chunk in bytes_iterator: + f.write(chunk) + + with open(path, "r") as f: + assert f.read() == file_content From af3eb0b0b12dda8f9e94a6fd29828831b2dc681a Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Thu, 14 Apr 2022 14:40:43 +0200 Subject: [PATCH 06/12] docs(api-docs): make iterator download documentation more generic Co-authored-by: Nejc Habjan --- docs/gl_objects/pipelines_and_jobs.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/gl_objects/pipelines_and_jobs.rst b/docs/gl_objects/pipelines_and_jobs.rst index 78cae51ba..901799ff6 100644 --- a/docs/gl_objects/pipelines_and_jobs.rst +++ b/docs/gl_objects/pipelines_and_jobs.rst @@ -274,12 +274,13 @@ You can also directly stream the output into a file, and unzip it afterwards:: subprocess.run(["unzip", "-bo", zipfn]) os.unlink(zipfn) -It is also possible to use the underlying iterator directly:: +Or, you can also use the underlying response iterator directly:: artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') -This can be used with FastAPI/Starlette StreamingResponse to forward a download from gitlab without having to download -the entire file server side first:: +This can be used with frameworks that expect an iterator (such as FastAPI/Starlette's +``StreamingResponse``) to forward a download from GitLab without having to download +the entire content server-side first:: @app.get("/download_artifact") def download_artifact(): From 4b31b517f107097253e47fb940c9705b53730481 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 25 Jun 2022 22:17:42 +0200 Subject: [PATCH 07/12] Revert "chore(deps): add conditional dependency on typing_extensions" This reverts commit 8cc4cfc0e78b8e1f430ed9e23089423cad3381ed. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cccf30dd7..c94a1d220 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ requests==2.27.1 requests-toolbelt==0.9.1 -typing_extensions; python_version<"3.8" \ No newline at end of file From 7e5f4eed642c1a7839ca98627062d8358be1554b Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 25 Jun 2022 22:17:42 +0200 Subject: [PATCH 08/12] Revert "feat(downloads): add conditional dependency on literal for python 3.7" This reverts commit efd8b486605bb38a5b0e8f5066eed675ed389836. --- gitlab/mixins.py | 7 +------ gitlab/utils.py | 8 +------- gitlab/v4/objects/artifacts.py | 8 +------- gitlab/v4/objects/files.py | 7 +------ gitlab/v4/objects/jobs.py | 18 +++++++++++------- gitlab/v4/objects/packages.py | 17 ++++++++++------- gitlab/v4/objects/projects.py | 7 +------ gitlab/v4/objects/repositories.py | 18 +++++++++++------- gitlab/v4/objects/snippets.py | 18 +++++++++++------- 9 files changed, 48 insertions(+), 60 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index 68388a410..f8b5a8a30 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -import sys from types import ModuleType from typing import ( Any, @@ -23,6 +22,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Type, @@ -38,11 +38,6 @@ from gitlab import types as g_types from gitlab import utils -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "GetMixin", "GetWithoutIdMixin", diff --git a/gitlab/utils.py b/gitlab/utils.py index 46b4ea773..0193a7562 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -16,19 +16,13 @@ # along with this program. If not, see . import pathlib -import sys import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Iterator, Optional, Type, Union +from typing import Any, Callable, Dict, Iterator, Literal, Optional, Type, Union import requests -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - class _StdoutStream: def __call__(self, chunk: Any) -> None: diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index d78efae21..cdcfda7c1 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,8 +2,7 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -import sys -from typing import Any, Callable, Iterator, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union import requests @@ -12,11 +11,6 @@ from gitlab import utils from gitlab.base import RESTManager, RESTObject -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = ["ProjectArtifact", "ProjectArtifactManager"] diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index 3476a18c0..1f4d98ccf 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,5 +1,4 @@ import base64 -import sys from typing import ( Any, Callable, @@ -7,6 +6,7 @@ Dict, Iterator, List, + Literal, Optional, TYPE_CHECKING, Union, @@ -27,11 +27,6 @@ UpdateMixin, ) -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "ProjectFile", "ProjectFileManager", diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index ef018ca3f..db278287a 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,5 +1,14 @@ -import sys -from typing import Any, Callable, cast, Dict, Iterator, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -9,11 +18,6 @@ from gitlab.base import RESTManager, RESTObject from gitlab.mixins import RefreshMixin, RetrieveMixin -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "ProjectJob", "ProjectJobManager", diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 60a06d8d3..1f8539558 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -4,9 +4,17 @@ https://docs.gitlab.com/ee/user/packages/generic_packages/ """ -import sys from pathlib import Path -from typing import Any, Callable, cast, Iterator, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Iterator, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -16,11 +24,6 @@ from gitlab.base import RESTManager, RESTObject from gitlab.mixins import DeleteMixin, GetMixin, ListMixin, ObjectDeleteMixin -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "GenericPackage", "GenericPackageManager", diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index 74053c13e..cd4aafd6c 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,4 +1,3 @@ -import sys from typing import ( Any, Callable, @@ -6,6 +5,7 @@ Dict, Iterator, List, + Literal, Optional, TYPE_CHECKING, Union, @@ -82,11 +82,6 @@ from .variables import ProjectVariableManager # noqa: F401 from .wikis import ProjectWikiManager # noqa: F401 -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "GroupProject", "GroupProjectManager", diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index b8af26979..9ea5e5237 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,8 +3,17 @@ Currently this module only contains repository-related methods for projects. """ -import sys -from typing import Any, Callable, Dict, Iterator, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -13,11 +22,6 @@ from gitlab import exceptions as exc from gitlab import utils -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - if TYPE_CHECKING: # When running mypy we use these as the base classes _RestObjectBase = gitlab.base.RESTObject diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 8c4dd7159..6d06fc7ed 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,5 +1,14 @@ -import sys -from typing import Any, Callable, cast, Iterator, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -13,11 +22,6 @@ from .discussions import ProjectSnippetDiscussionManager # noqa: F401 from .notes import ProjectSnippetNoteManager # noqa: F401 -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "Snippet", "SnippetManager", From 5a97cdb1e4a343a15aa69352a4f07d320e37bf95 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 25 Jun 2022 22:17:42 +0200 Subject: [PATCH 09/12] Revert "feat(downloads): allow streaming downloads access to response iterator" This reverts commit 4f9807f2fc137b81ef79c6f7d9a7d67c16f73a17. --- gitlab/mixins.py | 6 ++---- gitlab/utils.py | 9 +++------ gitlab/v4/cli.py | 3 +-- gitlab/v4/objects/artifacts.py | 15 +++++++-------- gitlab/v4/objects/files.py | 17 +++-------------- gitlab/v4/objects/jobs.py | 20 +++++--------------- gitlab/v4/objects/packages.py | 15 +++------------ gitlab/v4/objects/projects.py | 19 ++++--------------- gitlab/v4/objects/repositories.py | 20 +++++--------------- gitlab/v4/objects/snippets.py | 20 +++++--------------- 10 files changed, 38 insertions(+), 106 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index f8b5a8a30..1a3ff4dbf 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -20,9 +20,7 @@ Any, Callable, Dict, - Iterator, List, - Literal, Optional, Tuple, Type, @@ -659,10 +657,10 @@ class DownloadMixin(_RestObjectBase): def download( self, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Download the archive of a resource export. Args: diff --git a/gitlab/utils.py b/gitlab/utils.py index 0193a7562..197935549 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -19,7 +19,7 @@ import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Iterator, Literal, Optional, Type, Union +from typing import Any, Callable, Dict, Optional, Type, Union import requests @@ -32,18 +32,15 @@ def __call__(self, chunk: Any) -> None: def response_content( response: requests.Response, streamed: bool, - action: Optional[Union[Callable, Literal["iterator"]]], + action: Optional[Callable], chunk_size: int, -) -> Optional[Union[bytes, Iterator[Any]]]: +) -> Optional[bytes]: if streamed is False: return response.content if action is None: action = _StdoutStream() - if action == "iterator": - return response.iter_content(chunk_size=chunk_size) - for chunk in response.iter_content(chunk_size=chunk_size): if chunk: action(chunk) diff --git a/gitlab/v4/cli.py b/gitlab/v4/cli.py index f9349bdbd..6830b0874 100644 --- a/gitlab/v4/cli.py +++ b/gitlab/v4/cli.py @@ -19,7 +19,7 @@ import argparse import operator import sys -from typing import Any, Dict, Iterator, List, Optional, Type, TYPE_CHECKING, Union +from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING, Union import gitlab import gitlab.base @@ -123,7 +123,6 @@ def do_project_export_download(self) -> None: data = export_status.download() if TYPE_CHECKING: assert data is not None - assert not isinstance(data, Iterator) sys.stdout.buffer.write(data) except Exception as e: diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index cdcfda7c1..541e5e2f4 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,7 +2,7 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Optional, TYPE_CHECKING import requests @@ -32,7 +32,7 @@ def __call__( self, *args: Any, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: utils.warn( message=( "The project.artifacts() method is deprecated and will be removed in a " @@ -71,10 +71,10 @@ def download( ref_name: str, job: str, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Get the job artifacts archive from a specific tag or branch. Args: @@ -86,8 +86,7 @@ def download( `chunk_size` and each chunk is passed to `action` for treatment action: Callable responsible of dealing with chunk of - data. May also be the string "iterator" to directly return - the response iterator + data chunk_size: Size of each chunk **kwargs: Extra options to send to the server (e.g. sudo) @@ -116,10 +115,10 @@ def raw( artifact_path: str, job: str, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Download a single artifact file from a specific tag or branch from within the job's artifacts archive. diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index 1f4d98ccf..435e71b55 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,16 +1,5 @@ import base64 -from typing import ( - Any, - Callable, - cast, - Dict, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING import requests @@ -228,10 +217,10 @@ def raw( file_path: str, ref: str, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return the content of a file for a commit. Args: diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index db278287a..fbcb1fd40 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,14 +1,4 @@ -from typing import ( - Any, - Callable, - cast, - Dict, - Iterator, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Dict, Optional, TYPE_CHECKING, Union import requests @@ -126,10 +116,10 @@ def delete_artifacts(self, **kwargs: Any) -> None: def artifacts( self, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Get the job artifacts. Args: @@ -162,10 +152,10 @@ def artifact( self, path: str, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Get a single artifact file from within the job's artifacts archive. Args: diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 1f8539558..0461bdcd9 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -5,16 +5,7 @@ """ from pathlib import Path -from typing import ( - Any, - Callable, - cast, - Iterator, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Optional, TYPE_CHECKING, Union import requests @@ -112,10 +103,10 @@ def download( package_version: str, file_name: str, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Download a generic package. Args: diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index cd4aafd6c..81eb62496 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,15 +1,4 @@ -from typing import ( - Any, - Callable, - cast, - Dict, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING, Union import requests @@ -468,10 +457,10 @@ def snapshot( self, wiki: bool = False, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return a snapshot of the repository. Args: @@ -573,7 +562,7 @@ def artifact( self, *args: Any, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: utils.warn( message=( "The project.artifact() method is deprecated and will be " diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index 9ea5e5237..f2792b14e 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,17 +3,7 @@ Currently this module only contains repository-related methods for projects. """ -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union import requests @@ -117,10 +107,10 @@ def repository_raw_blob( self, sha: str, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return the raw file contents for a blob. Args: @@ -202,11 +192,11 @@ def repository_archive( self, sha: str = None, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, format: Optional[str] = None, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return an archive of the repository. Args: diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 6d06fc7ed..9d9dcc4e6 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,14 +1,4 @@ -from typing import ( - Any, - Callable, - cast, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, List, Optional, TYPE_CHECKING, Union import requests @@ -38,10 +28,10 @@ class Snippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObject): def content( self, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return the content of a snippet. Args: @@ -112,10 +102,10 @@ class ProjectSnippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObj def content( self, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return the content of a snippet. Args: From 8580a778aaf89f75c95f5bec727125f7c693d0b4 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 26 Jun 2022 00:17:18 +0200 Subject: [PATCH 10/12] feat(downloads): use iterator=True for returning response iterator for downloads Instead of having action="iterator", we can now do iterator=True to get the underlying response iterator when downloading things. --- gitlab/mixins.py | 8 ++++++-- gitlab/utils.py | 8 ++++++-- gitlab/v4/cli.py | 1 + gitlab/v4/objects/artifacts.py | 22 ++++++++++++++++------ gitlab/v4/objects/files.py | 19 ++++++++++++++++--- gitlab/v4/objects/jobs.py | 23 +++++++++++++++++------ gitlab/v4/objects/packages.py | 9 ++++++--- gitlab/v4/objects/projects.py | 25 +++++++++++++++++++++---- gitlab/v4/objects/repositories.py | 16 +++++++++++----- gitlab/v4/objects/snippets.py | 16 +++++++++++----- 10 files changed, 111 insertions(+), 36 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index 71ba8210c..14542e0a6 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -20,6 +20,7 @@ Any, Callable, Dict, + Iterator, List, Optional, Tuple, @@ -614,16 +615,19 @@ class DownloadMixin(_RestObjectBase): def download( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download the archive of a resource export. Args: streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -642,7 +646,7 @@ def download( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) class SubscribableMixin(_RestObjectBase): diff --git a/gitlab/utils.py b/gitlab/utils.py index bab670584..6acb86160 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -19,7 +19,7 @@ import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Optional, Tuple, Type, Union +from typing import Any, Callable, Dict, Iterator, Optional, Tuple, Type, Union import requests @@ -34,9 +34,13 @@ def __call__(self, chunk: Any) -> None: def response_content( response: requests.Response, streamed: bool, + iterator: bool, action: Optional[Callable], chunk_size: int, -) -> Optional[bytes]: +) -> Optional[Union[bytes, Iterator[Any]]]: + if iterator: + return response.iter_content(chunk_size=chunk_size) + if streamed is False: return response.content diff --git a/gitlab/v4/cli.py b/gitlab/v4/cli.py index 2b0d4ce72..ba2e788b7 100644 --- a/gitlab/v4/cli.py +++ b/gitlab/v4/cli.py @@ -127,6 +127,7 @@ def do_project_export_download(self) -> None: data = export_status.download() if TYPE_CHECKING: assert data is not None + assert isinstance(data, bytes) sys.stdout.buffer.write(data) except Exception as e: # pragma: no cover, cli.die is unit-tested diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index 541e5e2f4..f5f106d8b 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,7 +2,7 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -from typing import Any, Callable, Optional, TYPE_CHECKING +from typing import Any, Callable, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -40,10 +40,14 @@ def __call__( ), category=DeprecationWarning, ) - return self.download( + data = self.download( *args, **kwargs, ) + if TYPE_CHECKING: + assert data is not None + assert isinstance(data, bytes) + return data @exc.on_http_error(exc.GitlabDeleteError) def delete(self, **kwargs: Any) -> None: @@ -71,10 +75,11 @@ def download( ref_name: str, job: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get the job artifacts archive from a specific tag or branch. Args: @@ -85,6 +90,8 @@ def download( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -103,7 +110,7 @@ def download( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action( "ProjectArtifactManager", ("ref_name", "artifact_path", "job") @@ -115,10 +122,11 @@ def raw( artifact_path: str, job: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download a single artifact file from a specific tag or branch from within the job's artifacts archive. @@ -130,6 +138,8 @@ def raw( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -148,4 +158,4 @@ def raw( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index aa86704c9..2fd79fd54 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,5 +1,15 @@ import base64 -from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + List, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -220,10 +230,11 @@ def raw( file_path: str, ref: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a file for a commit. Args: @@ -232,6 +243,8 @@ def raw( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -252,7 +265,7 @@ def raw( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("ProjectFileManager", ("file_path", "ref")) @exc.on_http_error(exc.GitlabListError) diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index fbcb1fd40..850227725 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, cast, Dict, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, cast, Dict, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -116,16 +116,19 @@ def delete_artifacts(self, **kwargs: Any) -> None: def artifacts( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get the job artifacts. Args: streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -144,7 +147,7 @@ def artifacts( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("ProjectJob") @exc.on_http_error(exc.GitlabGetError) @@ -152,10 +155,11 @@ def artifact( self, path: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get a single artifact file from within the job's artifacts archive. Args: @@ -163,6 +167,8 @@ def artifact( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -181,13 +187,14 @@ def artifact( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("ProjectJob") @exc.on_http_error(exc.GitlabGetError) def trace( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, @@ -198,6 +205,8 @@ def trace( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -216,7 +225,9 @@ def trace( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return_value = utils.response_content(result, streamed, action, chunk_size) + return_value = utils.response_content( + result, streamed, iterator, action, chunk_size + ) if TYPE_CHECKING: assert isinstance(return_value, dict) return return_value diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 882cb1a5a..a82080167 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -5,7 +5,7 @@ """ from pathlib import Path -from typing import Any, Callable, cast, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, cast, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -103,10 +103,11 @@ def download( package_version: str, file_name: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download a generic package. Args: @@ -116,6 +117,8 @@ def download( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -132,7 +135,7 @@ def download( result = self.gitlab.http_get(path, streamed=streamed, raw=True, **kwargs) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) class GroupPackage(RESTObject): diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index f32cf2257..fc2aac3d5 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,4 +1,14 @@ -from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + List, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -466,10 +476,11 @@ def snapshot( self, wiki: bool = False, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return a snapshot of the repository. Args: @@ -477,6 +488,8 @@ def snapshot( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment. + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -495,7 +508,7 @@ def snapshot( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("Project", ("scope", "search")) @exc.on_http_error(exc.GitlabSearchError) @@ -579,7 +592,11 @@ def artifact( ), category=DeprecationWarning, ) - return self.artifacts.raw(*args, **kwargs) + data = self.artifacts.raw(*args, **kwargs) + if TYPE_CHECKING: + assert data is not None + assert isinstance(data, bytes) + return data class ProjectManager(CRUDMixin, RESTManager): diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index 5826d9d83..1f10473aa 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,7 +3,7 @@ Currently this module only contains repository-related methods for projects. """ -from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Dict, Iterator, List, Optional, TYPE_CHECKING, Union import requests @@ -107,10 +107,11 @@ def repository_raw_blob( self, sha: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the raw file contents for a blob. Args: @@ -118,6 +119,8 @@ def repository_raw_blob( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -136,7 +139,7 @@ def repository_raw_blob( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("Project", ("from_", "to")) @exc.on_http_error(exc.GitlabGetError) @@ -192,11 +195,12 @@ def repository_archive( self, sha: str = None, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, format: Optional[str] = None, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return an archive of the repository. Args: @@ -204,6 +208,8 @@ def repository_archive( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -228,7 +234,7 @@ def repository_archive( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("Project") @exc.on_http_error(exc.GitlabDeleteError) diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 597a3aaf0..aa46c7747 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, cast, List, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, cast, Iterator, List, Optional, TYPE_CHECKING, Union import requests @@ -29,16 +29,19 @@ class Snippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObject): def content( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a snippet. Args: streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment. + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -57,7 +60,7 @@ def content( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) class SnippetManager(CRUDMixin, RESTManager): @@ -103,16 +106,19 @@ class ProjectSnippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObj def content( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a snippet. Args: streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment. + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -131,7 +137,7 @@ def content( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) class ProjectSnippetManager(CRUDMixin, RESTManager): From ec61e736d87b26bc3a90b0fabc4b21788986f3e7 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 26 Jun 2022 00:19:18 +0200 Subject: [PATCH 11/12] test(packages): update tests for downloading with response iterator Adapted the existing tests for the new iterator=True argument --- tests/functional/api/test_packages.py | 6 ++---- tests/unit/test_utils.py | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/functional/api/test_packages.py b/tests/functional/api/test_packages.py index e009572c4..9f06439b4 100644 --- a/tests/functional/api/test_packages.py +++ b/tests/functional/api/test_packages.py @@ -53,8 +53,7 @@ def test_stream_generic_package(project): package_name=package_name, package_version=package_version, file_name=file_name, - streamed=True, - action="iterator", + iterator=True, ) assert isinstance(bytes_iterator, Iterator) @@ -90,8 +89,7 @@ def test_stream_generic_package_to_file(tmp_path, project): package_name=package_name, package_version=package_version, file_name=file_name, - streamed=True, - action="iterator", + iterator=True, ) with open(path, "wb") as f: diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c5f9f931d..74b48ae31 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -36,7 +36,9 @@ def test_response_content(capsys): ) resp = requests.get("https://example.com", stream=True) - utils.response_content(resp, streamed=True, action=None, chunk_size=1024) + utils.response_content( + resp, streamed=True, iterator=False, action=None, chunk_size=1024 + ) captured = capsys.readouterr() assert "test" in captured.out From 9f01ee59ef026758c7fe4ba0bb74bab4899e9353 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 26 Jun 2022 00:20:12 +0200 Subject: [PATCH 12/12] docs(api-docs): update artifact download iterator example Adapted the example for the new iterator=True argument --- docs/gl_objects/pipelines_and_jobs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/gl_objects/pipelines_and_jobs.rst b/docs/gl_objects/pipelines_and_jobs.rst index 901799ff6..f0bdd3a68 100644 --- a/docs/gl_objects/pipelines_and_jobs.rst +++ b/docs/gl_objects/pipelines_and_jobs.rst @@ -276,7 +276,7 @@ You can also directly stream the output into a file, and unzip it afterwards:: Or, you can also use the underlying response iterator directly:: - artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') + artifact_bytes_iterator = build_or_job.artifacts(iterator=True) This can be used with frameworks that expect an iterator (such as FastAPI/Starlette's ``StreamingResponse``) to forward a download from GitLab without having to download @@ -284,7 +284,7 @@ the entire content server-side first:: @app.get("/download_artifact") def download_artifact(): - artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') + artifact_bytes_iterator = build_or_job.artifacts(iterator=True) return StreamingResponse(artifact_bytes_iterator, media_type="application/zip") Delete all artifacts of a project that can be deleted:: pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy