diff --git a/.repo-metadata.json b/.repo-metadata.json index 62797084a..2cd2642fe 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -2,9 +2,9 @@ "name": "storage", "name_pretty": "Google Cloud Storage", "product_documentation": "https://cloud.google.com/storage", - "client_documentation": "https://googleapis.dev/python/storage/latest", + "client_documentation": "https://cloud.google.com/python/docs/reference/storage/latest", "issue_tracker": "https://issuetracker.google.com/savedsearches/559782", - "release_level": "ga", + "release_level": "stable", "language": "python", "library_type": "GAPIC_MANUAL", "repo": "googleapis/python-storage", @@ -12,5 +12,6 @@ "api_id": "storage.googleapis.com", "requires_billing": true, "default_version": "", - "codeowner_team": "@googleapis/cloud-storage-dpe" + "codeowner_team": "@googleapis/cloud-storage-dpe", + "api_shortname": "storage" } diff --git a/CHANGELOG.md b/CHANGELOG.md index fc7d6da38..d5839af78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [1.44.0](https://www.github.com/googleapis/python-storage/compare/v1.43.0...v1.44.0) (2022-01-05) + + +### Features + +* add raw_download kwarg to BlobReader ([#668](https://www.github.com/googleapis/python-storage/issues/668)) ([10cdad6](https://www.github.com/googleapis/python-storage/commit/10cdad630739a324ae0b16a3d14a67ca4c8a23c2)) + + +### Documentation + +* Describe code sample more specifically ([#660](https://www.github.com/googleapis/python-storage/issues/660)) ([0459cb4](https://www.github.com/googleapis/python-storage/commit/0459cb4e866696c46385a5ad72e2a85db810a36b)) +* refresh readme instructions ([#667](https://www.github.com/googleapis/python-storage/issues/667)) ([ceb9314](https://www.github.com/googleapis/python-storage/commit/ceb931403a755f2a0bdc20144287dbc4700c3360)) +* This is just a simple PR to better describe what the code is doing in the comments. ([0459cb4](https://www.github.com/googleapis/python-storage/commit/0459cb4e866696c46385a5ad72e2a85db810a36b)) +* use writeable streamin example for 'download_blob_to_file' ([#676](https://www.github.com/googleapis/python-storage/issues/676)) ([96092d4](https://www.github.com/googleapis/python-storage/commit/96092d4be36be478f9671e8940de4fd09cc6f7f0)) + ## [1.43.0](https://www.github.com/googleapis/python-storage/compare/v1.42.3...v1.43.0) (2021-11-15) diff --git a/README.rst b/README.rst index 358a28f29..0796bb05d 100644 --- a/README.rst +++ b/README.rst @@ -34,20 +34,22 @@ In order to use this library, you first need to go through the following steps: .. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project .. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project .. _Enable the Google Cloud Storage API.: https://cloud.google.com/storage -.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html +.. _Setup Authentication.: https://cloud.google.com/storage/docs/reference/libraries#setting_up_authentication Installation ~~~~~~~~~~~~ -Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to -create isolated Python environments. The basic problem it addresses is one of -dependencies and versions, and indirectly permissions. +`Set up a Python development environment`_ and install this library in a `venv`. +`venv`_ is a tool to create isolated Python environments. The basic problem it +addresses is one of dependencies and versions, and indirectly permissions. -With `virtualenv`_, it's possible to install this library without needing system +Make sure you're using Python 3.3 or later, which includes `venv`_ by default. +With `venv`, it's possible to install this library without needing system install permissions, and without clashing with the installed system dependencies. -.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ +.. _Set up a Python development environment: https://cloud.google.com/python/docs/setup +.. _`venv`: https://docs.python.org/3/library/venv.html Supported Python Versions @@ -68,10 +70,9 @@ Mac/Linux .. code-block:: console - pip install virtualenv - virtualenv - source /bin/activate - /bin/pip install google-cloud-storage + python -m venv env + source env/bin/activate + pip install google-cloud-storage Windows @@ -79,10 +80,9 @@ Windows .. code-block:: console - pip install virtualenv - virtualenv - \Scripts\activate - \Scripts\pip.exe install google-cloud-storage + py -m venv env + .\env\Scripts\activate + pip install google-cloud-storage Example Usage @@ -90,8 +90,13 @@ Example Usage .. code:: python + # Imports the Google Cloud client library from google.cloud import storage + + # Instantiates a client client = storage.Client() + + # Creates a new bucket and uploads an object new_bucket = client.create_bucket('new-bucket-id') new_blob = new_bucket.blob('remote/path/storage.txt') new_blob.upload_from_filename(filename='/local/path.txt') @@ -103,4 +108,12 @@ Example Usage blob = bucket.get_blob('remote/path/to/file.txt') print(blob.download_as_bytes()) blob.upload_from_string('New contents!') - + + +What's Next +~~~~~~~~~~~ + +Now that you've set up your Python client for Cloud Storage, +you can get started running `Storage samples.`_ + +.. _Storage samples.: https://github.com/googleapis/python-storage/tree/main/samples diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 2d7eccf25..75b22861e 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -3850,6 +3850,10 @@ def open( - ``timeout`` - ``retry`` + For downloads only, the following additional arguments are supported: + + - ``raw_download`` + For uploads only, the following additional arguments are supported: - ``content_type`` @@ -3877,7 +3881,7 @@ def open( >>> client = storage.Client() >>> bucket = client.bucket("bucket-name") - >>> blob = bucket.get_blob("blob-name.txt") + >>> blob = bucket.blob("blob-name.txt") >>> with blob.open("rt") as f: >>> print(f.read()) diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index bef05ea91..9d1d49af8 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -1065,7 +1065,7 @@ def download_blob_to_file( >>> bucket = client.get_bucket('my-bucket-name') >>> blob = storage.Blob('path/to/blob', bucket) - >>> with open('file-to-download-to') as file_obj: + >>> with open('file-to-download-to', 'w') as file_obj: >>> client.download_blob_to_file(blob, file_obj) # API request. @@ -1074,7 +1074,7 @@ def download_blob_to_file( >>> from google.cloud import storage >>> client = storage.Client() - >>> with open('file-to-download-to') as file_obj: + >>> with open('file-to-download-to', 'w') as file_obj: >>> client.download_blob_to_file( >>> 'gs://bucket_name/path/to/blob', file_obj) diff --git a/google/cloud/storage/fileio.py b/google/cloud/storage/fileio.py index 54d1577f4..95bb12b1f 100644 --- a/google/cloud/storage/fileio.py +++ b/google/cloud/storage/fileio.py @@ -35,6 +35,7 @@ "if_metageneration_not_match", "timeout", "retry", + "raw_download", } # Valid keyword arguments for upload methods. diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index 602d66b15..ccad14b45 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.43.0" +__version__ = "1.44.0" diff --git a/noxfile.py b/noxfile.py index 67bfa4eeb..3bb910dc2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -154,24 +154,30 @@ def system(session): @nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS) def conftest_retry(session): """Run the retry conformance test suite.""" - conformance_test_path = os.path.join("tests", "conformance.py") conformance_test_folder_path = os.path.join("tests", "conformance") - conformance_test_exists = os.path.exists(conformance_test_path) conformance_test_folder_exists = os.path.exists(conformance_test_folder_path) # Environment check: only run tests if found. - if not conformance_test_exists and not conformance_test_folder_exists: + if not conformance_test_folder_exists: session.skip("Conformance tests were not found") - session.install("pytest",) + # Install all test dependencies and pytest plugin to run tests in parallel. + # Then install this package in-place. + session.install("pytest", "pytest-xdist") session.install("-e", ".") + # Run #CPU processes in parallel if no test session arguments are passed in. + if session.posargs: + test_cmd = [ + "py.test", + "--quiet", + conformance_test_folder_path, + *session.posargs, + ] + else: + test_cmd = ["py.test", "-n", "auto", "--quiet", conformance_test_folder_path] + # Run py.test against the conformance tests. - if conformance_test_exists: - session.run("py.test", "--quiet", conformance_test_path, *session.posargs) - if conformance_test_folder_exists: - session.run( - "py.test", "--quiet", conformance_test_folder_path, *session.posargs - ) + session.run(*test_cmd) @nox.session(python=DEFAULT_PYTHON_VERSION) diff --git a/samples/README.md b/samples/README.md index bb48adc9b..2751bf722 100644 --- a/samples/README.md +++ b/samples/README.md @@ -21,8 +21,8 @@ Before running the samples, make sure you've followed the steps outlined in [Quick Start](https://github.com/googleapis/python-storage#quick-start). ### Authentication -This sample requires you to have authentication setup. Refer to the [Authentication Getting Started Guide](https://cloud.google.com/docs/authentication/getting-started) -for instructions on setting up credentials for applications. +Refer to the [Authentication Set Up Guide](https://cloud.google.com/storage/docs/reference/libraries#setting_up_authentication) +for more detailed instructions. ### Install Dependencies 1. Clone this repository and change to the sample directory you want to use. @@ -30,15 +30,12 @@ for instructions on setting up credentials for applications. git clone https://github.com/googleapis/python-storage.git ``` -2. Install [pip](https://pip.pypa.io/) and [virtualenv](https://virtualenv.pypa.io) if you do not already have them. You may want to refer to the [Python Development Environment Setup Guide](https://cloud.google.com/python/setup) for Google Cloud Platform for instructions. - -3. Create a virtualenv. Samples are compatible with Python 3.6+. +2. Activate a venv if you have not already from the [Quick Start](https://github.com/googleapis/python-storage#quick-start). ``` - virtualenv env - source env/bin/activate + source /bin/activate ``` -4. Install the dependencies needed to run the samples. +3. Install the dependencies needed to run the samples. ``` cd samples/snippets pip install -r requirements.txt @@ -49,6 +46,7 @@ for instructions on setting up credentials for applications. List of Samples * [Activate HMAC Key](#activate-hmac-key) +* [Batch Request](#batch-request) * [Add Bucket Conditional IAM Binding](#add-bucket-conditional-iam-binding) * [Add Bucket Default Owner](#add-bucket-default-owner) * [Add Bucket IAM Member](#add-bucket-iam-member) @@ -79,9 +77,11 @@ for instructions on setting up credentials for applications. * [Disable Requester Pays](#disable-requester-pays) * [Disable Uniform Bucket Level Access](#disable-uniform-bucket-level-access) * [Disable Versioning](#disable-versioning) +* [Download Byte Range](#download-byte-range) * [Download Encrypted File](#download-encrypted-file) * [Download File](#download-file) * [Download File Requester Pays](#download-file-requester-pays) +* [Download Into Memory](#download-into-memory) * [Download Public File](#download-public-file) * [Enable Bucket Lifecycle Management](#enable-bucket-lifecycle-management) * [Enable Default Event Based Hold](#enable-default-event-based-hold) @@ -139,11 +139,11 @@ for instructions on setting up credentials for applications. * [Set Metadata](#set-metadata) * [Set Public Access Prevention Enforced](#set-public-access-prevention-enforced) * [Set Public Access Prevention Inherited](#set-public-access-prevention-inherited) -* [Set Public Access Prevention Unspecified](#set-public-access-prevention-unspecified) * [Set Retention Policy](#set-retention-policy) * [Set Temporary Hold](#set-temporary-hold) * [Upload Encrypted File](#upload-encrypted-file) * [Upload File](#upload-file) +* [Upload From Memory](#upload-from-memory) * [Upload With KMS Key](#upload-with-kms-key) * [View Bucket IAM Members](#view-bucket-iam-members) @@ -158,6 +158,15 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_activate_hmac_key.py ` +----- +### Batch Request +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_batch_request.py,samples/README.md) + +View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_batch_request.py). To run this sample: + + +`python storage_batch_request.py ` + ----- ### Add Bucket Conditional IAM Binding @@ -429,6 +438,15 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_disable_versioning.py ` +----- +### Download Byte Range +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_download_byte_range.py,samples/README.md) + +View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_download_byte_range.py). To run this sample: + + +`python storage_download_byte_range.py <>BASE64_ENCRYPTION_KEY` + ----- ### Download Encrypted File [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_download_encrypted_file.py,samples/README.md) @@ -456,6 +474,15 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_download_file_requester_pays.py ` +----- +### Download Into Memory +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_download_into_memory.py,samples/README.md) + +View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_download_into_memory.py). To run this sample: + + +`python storage_download_into_memory.py ` + ----- ### Download Public File [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_download_public_file.py,samples/README.md) @@ -969,15 +996,6 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_set_public_access_prevention_inherited.py ` ------ -### Set Public Access Prevention Unspecified -[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_set_public_access_prevention_unspecified.py,samples/README.md) - -View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_set_public_access_prevention_unspecified.py). To run this sample: - - -`python storage_set_public_access_prevention_unspecified.py ` - ----- ### Set Retention Policy [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_set_retention_policy.py,samples/README.md) @@ -1014,6 +1032,15 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_upload_file.py ` +----- +### Upload From Memory +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_upload_from_memory.py,samples/README.md) + +View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_upload_from_memory.py). To run this sample: + + +`python storage_upload_from_memory.py ` + ----- ### Upload With KMS Key [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_upload_with_kms_key.py,samples/README.md) diff --git a/samples/snippets/public_access_prevention_test.py b/samples/snippets/public_access_prevention_test.py index 40d3924b2..558a4ef15 100644 --- a/samples/snippets/public_access_prevention_test.py +++ b/samples/snippets/public_access_prevention_test.py @@ -12,15 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest import storage_get_public_access_prevention import storage_set_public_access_prevention_enforced import storage_set_public_access_prevention_inherited -import storage_set_public_access_prevention_unspecified -@pytest.mark.skip(reason="Inconsistent due to unspecified->inherited change") def test_get_public_access_prevention(bucket, capsys): short_name = storage_get_public_access_prevention short_name.get_public_access_prevention(bucket.name) @@ -35,14 +32,6 @@ def test_set_public_access_prevention_enforced(bucket, capsys): assert f"Public access prevention is set to enforced for {bucket.name}." in out -@pytest.mark.skip(reason="Inconsistent due to unspecified->inherited change") -def test_set_public_access_prevention_unspecified(bucket, capsys): - short_name = storage_set_public_access_prevention_unspecified - short_name.set_public_access_prevention_unspecified(bucket.name) - out, _ = capsys.readouterr() - assert f"Public access prevention is 'unspecified' for {bucket.name}." in out - - def test_set_public_access_prevention_inherited(bucket, capsys): short_name = storage_set_public_access_prevention_inherited short_name.set_public_access_prevention_inherited(bucket.name) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index addda1960..240aa5070 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-pubsub==2.9.0 -google-cloud-storage==1.42.3 +google-cloud-storage==1.43.0 pandas==1.3.4; python_version > '3.6' pandas==1.1.5; python_version < '3.7' diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index dd8e6aeaf..7c0a5b91d 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -23,6 +23,7 @@ import requests import storage_add_bucket_label +import storage_batch_request import storage_bucket_delete_default_kms_key import storage_change_default_storage_class import storage_change_file_storage_class @@ -37,7 +38,9 @@ import storage_delete_file_archived_generation import storage_disable_bucket_lifecycle_management import storage_disable_versioning +import storage_download_byte_range import storage_download_file +import storage_download_into_memory import storage_download_public_file import storage_enable_bucket_lifecycle_management import storage_enable_versioning @@ -62,6 +65,7 @@ import storage_set_bucket_default_kms_key import storage_set_metadata import storage_upload_file +import storage_upload_from_memory import storage_upload_with_kms_key KMS_KEY = os.environ["CLOUD_KMS_KEY"] @@ -189,6 +193,15 @@ def test_upload_blob(test_bucket): ) +def test_upload_blob_from_memory(test_bucket, capsys): + storage_upload_from_memory.upload_blob_from_memory( + test_bucket.name, "Hello, is it me you're looking for?", "test_upload_blob" + ) + out, _ = capsys.readouterr() + + assert "Hello, is it me you're looking for?" in out + + def test_upload_blob_with_kms(test_bucket): with tempfile.NamedTemporaryFile() as source_file: source_file.write(b"test") @@ -200,6 +213,14 @@ def test_upload_blob_with_kms(test_bucket): assert kms_blob.kms_key_name.startswith(KMS_KEY) +def test_download_byte_range(test_blob): + with tempfile.NamedTemporaryFile() as dest_file: + storage_download_byte_range.download_byte_range( + test_blob.bucket.name, test_blob.name, 0, 4, dest_file.name + ) + assert dest_file.read() == b'Hello' + + def test_download_blob(test_blob): with tempfile.NamedTemporaryFile() as dest_file: storage_download_file.download_blob( @@ -209,6 +230,15 @@ def test_download_blob(test_blob): assert dest_file.read() +def test_download_blob_into_memory(test_blob, capsys): + storage_download_into_memory.download_blob_into_memory( + test_blob.bucket.name, test_blob.name + ) + out, _ = capsys.readouterr() + + assert "Hello, is it me you're looking for?" in out + + def test_blob_metadata(test_blob, capsys): storage_get_metadata.blob_metadata(test_blob.bucket.name, test_blob.name) out, _ = capsys.readouterr() @@ -509,3 +539,17 @@ def test_storage_configure_retries(test_blob, capsys): assert "The following library method is customized to be retried" in out assert "_should_retry" in out assert "initial=1.5, maximum=45.0, multiplier=1.2, deadline=500.0" in out + + +def test_batch_request(test_bucket): + blob1 = test_bucket.blob("b/1.txt") + blob2 = test_bucket.blob("b/2.txt") + blob1.upload_from_string("hello world") + blob2.upload_from_string("hello world") + + storage_batch_request.batch_request(test_bucket.name, "b/") + blob1.reload() + blob2.reload() + + assert blob1.metadata.get("your-metadata-key") == "your-metadata-value" + assert blob2.metadata.get("your-metadata-key") == "your-metadata-value" diff --git a/samples/snippets/storage_batch_request.py b/samples/snippets/storage_batch_request.py new file mode 100644 index 000000000..863fc09cd --- /dev/null +++ b/samples/snippets/storage_batch_request.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that uses a batch request. +This sample is used on this page: + https://cloud.google.com/storage/docs/batch +For more information, see README.md. +""" + +# [START storage_batch_request] + +from google.cloud import storage + + +def batch_request(bucket_name, prefix=None): + """Use a batch request to patch a list of objects with the given prefix in a bucket.""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + # The prefix of the object paths + # prefix = "directory-prefix/" + + client = storage.Client() + bucket = client.bucket(bucket_name) + + # Accumulate in a list the objects with a given prefix. + blobs_to_patch = [blob for blob in bucket.list_blobs(prefix=prefix)] + + # Use a batch context manager to edit metadata in the list of blobs. + # The batch request is sent out when the context manager closes. + # No more than 100 calls should be included in a single batch request. + with client.batch(): + for blob in blobs_to_patch: + metadata = {"your-metadata-key": "your-metadata-value"} + blob.metadata = metadata + blob.patch() + + print( + f"Batch request edited metadata for all objects with the given prefix in {bucket.name}." + ) + + +# [END storage_batch_request] + +if __name__ == "__main__": + batch_request(bucket_name=sys.argv[1], prefix=sys.argv[2]) diff --git a/samples/snippets/storage_create_bucket_class_location.py b/samples/snippets/storage_create_bucket_class_location.py index 64c2652d7..51fa86440 100644 --- a/samples/snippets/storage_create_bucket_class_location.py +++ b/samples/snippets/storage_create_bucket_class_location.py @@ -21,7 +21,10 @@ def create_bucket_class_location(bucket_name): - """Create a new bucket in specific location with storage class""" + """ + Create a new bucket in the US region with the coldline storage + class + """ # bucket_name = "your-new-bucket-name" storage_client = storage.Client() diff --git a/samples/snippets/storage_download_byte_range.py b/samples/snippets/storage_download_byte_range.py new file mode 100644 index 000000000..e6143a04f --- /dev/null +++ b/samples/snippets/storage_download_byte_range.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_download_byte_range] +from google.cloud import storage + + +def download_byte_range( + bucket_name, source_blob_name, start_byte, end_byte, destination_file_name +): + """Downloads a blob from the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your GCS object + # source_blob_name = "storage-object-name" + + # The starting byte at which to begin the download + # start_byte = 0 + + # The ending byte at which to end the download + # end_byte = 20 + + # The path to which the file should be downloaded + # destination_file_name = "local/path/to/file" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Construct a client side representation of a blob. + # Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve + # any content from Google Cloud Storage. As we don't need additional data, + # using `Bucket.blob` is preferred here. + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name, start=start_byte, end=end_byte) + + print( + "Downloaded bytes {} to {} of object {} from bucket {} to local file {}.".format( + start_byte, end_byte, source_blob_name, bucket_name, destination_file_name + ) + ) + + +# [END storage_download_byte_range] + +if __name__ == "__main__": + download_byte_range( + bucket_name=sys.argv[1], + source_blob_name=sys.argv[2], + start_byte=sys.argv[3], + end_byte=sys.argv[4], + destination_file_name=sys.argv[5], + ) diff --git a/samples/snippets/storage_download_into_memory.py b/samples/snippets/storage_download_into_memory.py new file mode 100644 index 000000000..453a13e21 --- /dev/null +++ b/samples/snippets/storage_download_into_memory.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_file_download_into_memory] +from google.cloud import storage + + +def download_blob_into_memory(bucket_name, blob_name): + """Downloads a blob into memory.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your GCS object + # blob_name = "storage-object-name" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Construct a client side representation of a blob. + # Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve + # any content from Google Cloud Storage. As we don't need additional data, + # using `Bucket.blob` is preferred here. + blob = bucket.blob(blob_name) + contents = blob.download_as_string() + + print( + "Downloaded storage object {} from bucket {} as the following string: {}.".format( + blob_name, bucket_name, contents + ) + ) + + +# [END storage_file_download_into_memory] + +if __name__ == "__main__": + download_blob_into_memory( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + ) diff --git a/samples/snippets/storage_set_public_access_prevention_unspecified.py b/samples/snippets/storage_set_public_access_prevention_unspecified.py deleted file mode 100644 index ae2c4701c..000000000 --- a/samples/snippets/storage_set_public_access_prevention_unspecified.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -# [START storage_set_public_access_prevention_unspecified] -from google.cloud import storage -from google.cloud.storage.constants import PUBLIC_ACCESS_PREVENTION_UNSPECIFIED - - -def set_public_access_prevention_unspecified(bucket_name): - """Sets the public access prevention status to unspecified, so that the bucket inherits its setting from its parent project.""" - # The ID of your GCS bucket - # bucket_name = "my-bucket" - - storage_client = storage.Client() - bucket = storage_client.get_bucket(bucket_name) - - bucket.iam_configuration.public_access_prevention = ( - PUBLIC_ACCESS_PREVENTION_UNSPECIFIED - ) - bucket.patch() - - print(f"Public access prevention is 'unspecified' for {bucket.name}.") - - -# [END storage_set_public_access_prevention_unspecified] - -if __name__ == "__main__": - set_public_access_prevention_unspecified(bucket_name=sys.argv[1]) diff --git a/samples/snippets/storage_upload_from_memory.py b/samples/snippets/storage_upload_from_memory.py new file mode 100644 index 000000000..e5f61ff93 --- /dev/null +++ b/samples/snippets/storage_upload_from_memory.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_file_upload_from_memory] +from google.cloud import storage + + +def upload_blob_from_memory(bucket_name, contents, destination_blob_name): + """Uploads a file to the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The contents to upload to the file + # contents = "these are my contents" + # The ID of your GCS object + # destination_blob_name = "storage-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + blob.upload_from_string(contents) + + print( + "{} with contents {} uploaded to {}.".format( + destination_blob_name, contents, destination_blob_name + ) + ) + + +# [END storage_file_upload_from_memory] + +if __name__ == "__main__": + upload_blob_from_memory( + bucket_name=sys.argv[1], + contents=sys.argv[2], + destination_blob_name=sys.argv[3], + ) diff --git a/tests/conformance/retry_conformance_testing.md b/tests/conformance/README.md similarity index 84% rename from tests/conformance/retry_conformance_testing.md rename to tests/conformance/README.md index 22b7c1952..3b5fa8884 100644 --- a/tests/conformance/retry_conformance_testing.md +++ b/tests/conformance/README.md @@ -27,3 +27,11 @@ To run the test suite locally: ```bash nox -s conftest_retry-3.8 ``` + +To run a single test locally: + +Single test names are displayed as "test-S{scenario_id}-{method}-{client-library-method-name}-{instructions index}", such as `test-S1-storage.buckets.get-bucket_reload-1` + +```bash +nox -re conftest_retry-3.8 -- -k +``` diff --git a/tests/unit/test_fileio.py b/tests/unit/test_fileio.py index bf017e44f..d71103707 100644 --- a/tests/unit/test_fileio.py +++ b/tests/unit/test_fileio.py @@ -109,6 +109,24 @@ def read_from_fake_data(start=0, end=None, **_): reader.close() + def test_read_with_raw_download(self): + blob = mock.Mock() + + def read_from_fake_data(start=0, end=None, **_): + return TEST_BINARY_DATA[start:end] + + blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data) + download_kwargs = {"raw_download": True} + reader = self._make_blob_reader(blob, chunk_size=8, **download_kwargs) + + # Read and trigger the first download of chunk_size. + self.assertEqual(reader.read(1), TEST_BINARY_DATA[0:1]) + blob.download_as_bytes.assert_called_once_with( + start=0, end=8, checksum=None, retry=DEFAULT_RETRY, raw_download=True + ) + + reader.close() + def test_retry_passed_through(self): blob = mock.Mock() pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy