feat: add support for soft delete (#1229)

* feat: add support for soft delete

* add restore, get object, list_objects, unit tests

* integration test

* update restore_blob

* SoftDeletePolicy data class

* update docstrings; address comments

Author: cojenco

google/cloud/storage/_helpers.py

@@ -225,6 +225,7 @@ def reload(
         if_metageneration_not_match=None,
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
+        soft_deleted=None,
     ):
         """Reload properties from Cloud Storage.
 
@@ -270,6 +271,13 @@ def reload(
         :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
         :param retry:
             (Optional) How to retry the RPC. See: :ref:`configuring_retries`
+
+        :type soft_deleted: bool
+        :param soft_deleted:
+            (Optional) If True, looks for a soft-deleted object. Will only return
+            the object metadata if the object exists and is in a soft-deleted state.
+            :attr:`generation` is required to be set on the blob if ``soft_deleted`` is set to True.
+            See: https://cloud.google.com/storage/docs/soft-delete
         """
         client = self._require_client(client)
         query_params = self._query_params
@@ -283,6 +291,8 @@ def reload(
             if_metageneration_match=if_metageneration_match,
             if_metageneration_not_match=if_metageneration_not_match,
         )
+        if soft_deleted is not None:
+            query_params["softDeleted"] = soft_deleted
         headers = self._encryption_headers()
         _add_etag_match_headers(
             headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match

google/cloud/storage/blob.py

@@ -650,6 +650,7 @@ def exists(
         if_metageneration_not_match=None,
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
+        soft_deleted=None,
     ):
         """Determines whether or not this blob exists.
 
@@ -694,6 +695,13 @@ def exists(
         :param retry:
             (Optional) How to retry the RPC. See: :ref:`configuring_retries`
 
+        :type soft_deleted: bool
+        :param soft_deleted:
+            (Optional) If True, looks for a soft-deleted object. Will only return True
+            if the object exists and is in a soft-deleted state.
+            :attr:`generation` is required to be set on the blob if ``soft_deleted`` is set to True.
+            See: https://cloud.google.com/storage/docs/soft-delete
+
         :rtype: bool
         :returns: True if the blob exists in Cloud Storage.
         """
@@ -702,6 +710,8 @@ def exists(
         # minimize the returned payload.
         query_params = self._query_params
         query_params["fields"] = "name"
+        if soft_deleted is not None:
+            query_params["softDeleted"] = soft_deleted
 
         _add_generation_match_parameters(
             query_params,
@@ -4700,6 +4710,32 @@ def retention(self):
         info = self._properties.get("retention", {})
         return Retention.from_api_repr(info, self)
 
+    @property
+    def soft_delete_time(self):
+        """If this object has been soft-deleted, returns the time at which it became soft-deleted.
+
+        :rtype: :class:`datetime.datetime` or ``NoneType``
+        :returns:
+            (readonly) The time that the object became soft-deleted.
+             Note this property is only set for soft-deleted objects.
+        """
+        soft_delete_time = self._properties.get("softDeleteTime")
+        if soft_delete_time is not None:
+            return _rfc3339_nanos_to_datetime(soft_delete_time)
+
+    @property
+    def hard_delete_time(self):
+        """If this object has been soft-deleted, returns the time at which it will be permanently deleted.
+
+        :rtype: :class:`datetime.datetime` or ``NoneType``
+        :returns:
+            (readonly) The time that the object will be permanently deleted.
+            Note this property is only set for soft-deleted objects.
+        """
+        hard_delete_time = self._properties.get("hardDeleteTime")
+        if hard_delete_time is not None:
+            return _rfc3339_nanos_to_datetime(hard_delete_time)
+
 
 def _get_host_name(connection):
     """Returns the host name from the given connection.

google/cloud/storage/bucket.py

@@ -1188,6 +1188,7 @@ def get_blob(
         if_metageneration_not_match=None,
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
+        soft_deleted=None,
         **kwargs,
     ):
         """Get a blob object by name.
@@ -1248,6 +1249,13 @@ def get_blob(
         :param retry:
             (Optional) How to retry the RPC. See: :ref:`configuring_retries`
 
+        :type soft_deleted: bool
+        :param soft_deleted:
+            (Optional) If True, looks for a soft-deleted object. Will only return
+            the object metadata if the object exists and is in a soft-deleted state.
+            Object ``generation`` is required if ``soft_deleted`` is set to True.
+            See: https://cloud.google.com/storage/docs/soft-delete
+
         :param kwargs: Keyword arguments to pass to the
                        :class:`~google.cloud.storage.blob.Blob` constructor.
 
@@ -1275,6 +1283,7 @@ def get_blob(
                 if_metageneration_match=if_metageneration_match,
                 if_metageneration_not_match=if_metageneration_not_match,
                 retry=retry,
+                soft_deleted=soft_deleted,
             )
         except NotFound:
             return None
@@ -1297,6 +1306,7 @@ def list_blobs(
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
         match_glob=None,
+        soft_deleted=None,
     ):
         """Return an iterator used to find blobs in the bucket.
 
@@ -1378,6 +1388,13 @@ def list_blobs(
             The string value must be UTF-8 encoded. See:
             https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob
 
+        :type soft_deleted: bool
+        :param soft_deleted:
+            (Optional) If true, only soft-deleted objects will be listed as distinct results in order of increasing
+            generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
+            Note ``soft_deleted`` and ``versions`` cannot be set to True simultaneously. See:
+            https://cloud.google.com/storage/docs/soft-delete
+
         :rtype: :class:`~google.api_core.page_iterator.Iterator`
         :returns: Iterator of all :class:`~google.cloud.storage.blob.Blob`
                   in this bucket matching the arguments.
@@ -1398,6 +1415,7 @@ def list_blobs(
             timeout=timeout,
             retry=retry,
             match_glob=match_glob,
+            soft_deleted=soft_deleted,
         )
 
     def list_notifications(
@@ -2060,6 +2078,110 @@ def rename_blob(
             )
         return new_blob
 
+    def restore_blob(
+        self,
+        blob_name,
+        client=None,
+        generation=None,
+        copy_source_acl=None,
+        projection=None,
+        if_generation_match=None,
+        if_generation_not_match=None,
+        if_metageneration_match=None,
+        if_metageneration_not_match=None,
+        timeout=_DEFAULT_TIMEOUT,
+        retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED,
+    ):
+        """Restores a soft-deleted object.
+
+        If :attr:`user_project` is set on the bucket, bills the API request to that project.
+
+        See [API reference docs](https://cloud.google.com/storage/docs/json_api/v1/objects/restore)
+
+        :type blob_name: str
+        :param blob_name: The name of the blob to be restored.
+
+        :type client: :class:`~google.cloud.storage.client.Client`
+        :param client: (Optional) The client to use. If not passed, falls back
+                       to the ``client`` stored on the current bucket.
+
+        :type generation: long
+        :param generation: (Optional) If present, selects a specific revision of this object.
+
+        :type copy_source_acl: bool
+        :param copy_source_acl: (Optional) If true, copy the soft-deleted object's access controls.
+
+        :type projection: str
+        :param projection: (Optional) Specifies the set of properties to return.
+                           If used, must be 'full' or 'noAcl'.
+
+        :type if_generation_match: long
+        :param if_generation_match:
+            (Optional) See :ref:`using-if-generation-match`
+
+        :type if_generation_not_match: long
+        :param if_generation_not_match:
+            (Optional) See :ref:`using-if-generation-not-match`
+
+        :type if_metageneration_match: long
+        :param if_metageneration_match:
+            (Optional) See :ref:`using-if-metageneration-match`
+
+        :type if_metageneration_not_match: long
+        :param if_metageneration_not_match:
+            (Optional) See :ref:`using-if-metageneration-not-match`
+
+        :type timeout: float or tuple
+        :param timeout:
+            (Optional) The amount of time, in seconds, to wait
+            for the server response.  See: :ref:`configuring_timeouts`
+
+        :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
+        :param retry:
+            (Optional) How to retry the RPC.
+            The default value is ``DEFAULT_RETRY_IF_GENERATION_SPECIFIED``, which
+            only restore operations with ``if_generation_match`` or ``generation`` set
+            will be retried.
+
+            Users can configure non-default retry behavior. A ``None`` value will
+            disable retries. A ``DEFAULT_RETRY`` value will enable retries
+            even if restore operations are not guaranteed to be idempotent.
+            See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout).
+
+        :rtype: :class:`google.cloud.storage.blob.Blob`
+        :returns: The restored Blob.
+        """
+        client = self._require_client(client)
+        query_params = {}
+
+        if self.user_project is not None:
+            query_params["userProject"] = self.user_project
+        if generation is not None:
+            query_params["generation"] = generation
+        if copy_source_acl is not None:
+            query_params["copySourceAcl"] = copy_source_acl
+        if projection is not None:
+            query_params["projection"] = projection
+
+        _add_generation_match_parameters(
+            query_params,
+            if_generation_match=if_generation_match,
+            if_generation_not_match=if_generation_not_match,
+            if_metageneration_match=if_metageneration_match,
+            if_metageneration_not_match=if_metageneration_not_match,
+        )
+
+        blob = Blob(bucket=self, name=blob_name)
+        api_response = client._post_resource(
+            f"{blob.path}/restore",
+            None,
+            query_params=query_params,
+            timeout=timeout,
+            retry=retry,
+        )
+        blob._set_properties(api_response)
+        return blob
+
     @property
     def cors(self):
         """Retrieve or set CORS policies configured for this bucket.
@@ -2227,6 +2349,18 @@ def iam_configuration(self):
         info = self._properties.get("iamConfiguration", {})
         return IAMConfiguration.from_api_repr(info, self)
 
+    @property
+    def soft_delete_policy(self):
+        """Retrieve the soft delete policy for this bucket.
+
+        See https://cloud.google.com/storage/docs/soft-delete
+
+        :rtype: :class:`SoftDeletePolicy`
+        :returns: an instance for managing the bucket's soft delete policy.
+        """
+        policy = self._properties.get("softDeletePolicy", {})
+        return SoftDeletePolicy.from_api_repr(policy, self)
+
     @property
     def lifecycle_rules(self):
         """Retrieve or set lifecycle rules configured for this bucket.
@@ -3432,6 +3566,102 @@ def generate_signed_url(
         )
 
 
+class SoftDeletePolicy(dict):
+    """Map a bucket's soft delete policy.
+
+    See https://cloud.google.com/storage/docs/soft-delete
+
+    :type bucket: :class:`Bucket`
+    :param bucket: Bucket for which this instance is the policy.
+
+    :type retention_duration_seconds: int
+    :param retention_duration_seconds:
+        (Optional) The period of time in seconds that soft-deleted objects in the bucket
+        will be retained and cannot be permanently deleted.
+
+    :type effective_time: :class:`datetime.datetime`
+    :param effective_time:
+        (Optional) When the bucket's soft delete policy is effective.
+        This value should normally only be set by the back-end API.
+    """
+
+    def __init__(self, bucket, **kw):
+        data = {}
+        retention_duration_seconds = kw.get("retention_duration_seconds")
+        data["retentionDurationSeconds"] = retention_duration_seconds
+
+        effective_time = kw.get("effective_time")
+        if effective_time is not None:
+            effective_time = _datetime_to_rfc3339(effective_time)
+        data["effectiveTime"] = effective_time
+
+        super().__init__(data)
+        self._bucket = bucket
+
+    @classmethod
+    def from_api_repr(cls, resource, bucket):
+        """Factory:  construct instance from resource.
+
+        :type resource: dict
+        :param resource: mapping as returned from API call.
+
+        :type bucket: :class:`Bucket`
+        :params bucket: Bucket for which this instance is the policy.
+
+        :rtype: :class:`SoftDeletePolicy`
+        :returns: Instance created from resource.
+        """
+        instance = cls(bucket)
+        instance.update(resource)
+        return instance
+
+    @property
+    def bucket(self):
+        """Bucket for which this instance is the policy.
+
+        :rtype: :class:`Bucket`
+        :returns: the instance's bucket.
+        """
+        return self._bucket
+
+    @property
+    def retention_duration_seconds(self):
+        """Get the retention duration of the bucket's soft delete policy.
+
+        :rtype: int or ``NoneType``
+        :returns: The period of time in seconds that soft-deleted objects in the bucket
+                  will be retained and cannot be permanently deleted; Or ``None`` if the
+                  property is not set.
+        """
+        duration = self.get("retentionDurationSeconds")
+        if duration is not None:
+            return int(duration)
+
+    @retention_duration_seconds.setter
+    def retention_duration_seconds(self, value):
+        """Set the retention duration of the bucket's soft delete policy.
+
+        :type value: int
+        :param value:
+            The period of time in seconds that soft-deleted objects in the bucket
+            will be retained and cannot be permanently deleted.
+        """
+        self["retentionDurationSeconds"] = value
+        self.bucket._patch_property("softDeletePolicy", self)
+
+    @property
+    def effective_time(self):
+        """Get the effective time of the bucket's soft delete policy.
+
+        :rtype: datetime.datetime or ``NoneType``
+        :returns: point-in time at which the bucket's soft delte policy is
+                  effective, or ``None`` if the property is not set.
+        """
+        timestamp = self.get("effectiveTime")
+        if timestamp is not None:
+            return _rfc3339_nanos_to_datetime(timestamp)
+
+
 def _raise_if_len_differs(expected_len, **generation_match_args):
     """
     Raise an error if any generation match argument