From 7492d8878f5b2b0786f991ea3697e954c4e95520 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 11 Sep 2026 21:40:41 +0000 Subject: [PATCH 01/43] [DRAFT] feat: add support for resumable uploads --- .../google/api_core/exceptions.py | 24 + .../api_core/resumable_transfer/__init__.py | 53 + .../api_core/resumable_transfer/common.py | 90 ++ .../api_core/resumable_transfer/upload.py | 907 ++++++++++++++++ .../resumable_transfer/upload_async.py | 854 +++++++++++++++ .../resumable_transfer/upload_state.py | 310 ++++++ .../asyncio/test_resumable_transfer_async.py | 992 ++++++++++++++++++ .../tests/unit/test_resumable_transfer.py | 730 +++++++++++++ 8 files changed, 3960 insertions(+) create mode 100644 packages/google-api-core/google/api_core/resumable_transfer/__init__.py create mode 100644 packages/google-api-core/google/api_core/resumable_transfer/common.py create mode 100644 packages/google-api-core/google/api_core/resumable_transfer/upload.py create mode 100644 packages/google-api-core/google/api_core/resumable_transfer/upload_async.py create mode 100644 packages/google-api-core/google/api_core/resumable_transfer/upload_state.py create mode 100644 packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py create mode 100644 packages/google-api-core/tests/unit/test_resumable_transfer.py diff --git a/packages/google-api-core/google/api_core/exceptions.py b/packages/google-api-core/google/api_core/exceptions.py index df3e54e8f223..d86c1c409f4a 100644 --- a/packages/google-api-core/google/api_core/exceptions.py +++ b/packages/google-api-core/google/api_core/exceptions.py @@ -446,6 +446,30 @@ class AsyncRestUnsupportedParameterError(NotImplementedError): pass +class TransferStalledError(GoogleAPICallError): + """Raised when upload throughput stays below minimum rate past stall timeout.""" + + pass + + +class UnseekableStreamError(GoogleAPICallError): + """Raised when server recovery requires rewinding a non-seekable stream.""" + + pass + + +class UploadCancelledError(GoogleAPICallError): + """Raised when the upload is cancelled by the client or server.""" + + pass + + +class MissingStatusHeaderError(GoogleAPICallError): + """Raised when server response lacks the required X-Goog-Upload-Status header.""" + + pass + + def exception_class_for_http_status(status_code): """Return the exception class for a specific HTTP status code. diff --git a/packages/google-api-core/google/api_core/resumable_transfer/__init__.py b/packages/google-api-core/google/api_core/resumable_transfer/__init__.py new file mode 100644 index 000000000000..34217378f70a --- /dev/null +++ b/packages/google-api-core/google/api_core/resumable_transfer/__init__.py @@ -0,0 +1,53 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Resumable transfer library for Google APIs.""" + +from google.api_core.exceptions import ( + MissingStatusHeaderError, + TransferStalledError, + UnseekableStreamError, + UploadCancelledError, +) +from google.api_core.resumable_transfer.common import ( + DEFAULT_CHUNK_SIZE, + Command, + ProgressState, + Status, + UploadProgress, +) +from google.api_core.resumable_transfer.upload import ( + ResumableUploadConfig, + ResumableUploadSession, +) +from google.api_core.resumable_transfer.upload_async import ( + AsyncResumableUploadSession, + AsyncUploadOperation, +) + +__all__ = [ + "Command", + "DEFAULT_CHUNK_SIZE", + "MissingStatusHeaderError", + "ProgressState", + "Status", + "TransferStalledError", + "UnseekableStreamError", + "UploadCancelledError", + "UploadProgress", + "ResumableUploadConfig", + "ResumableUploadSession", + "AsyncResumableUploadSession", + "AsyncUploadOperation", +] diff --git a/packages/google-api-core/google/api_core/resumable_transfer/common.py b/packages/google-api-core/google/api_core/resumable_transfer/common.py new file mode 100644 index 000000000000..cc0046f0d6ca --- /dev/null +++ b/packages/google-api-core/google/api_core/resumable_transfer/common.py @@ -0,0 +1,90 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common constants and headers for Resumable Upload protocol.""" + +import dataclasses +import enum +from typing import Optional + +# Default chunk size: 10 MiB +DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024 + +# Protocol Headers +HEADER_PROTOCOL = "X-Goog-Upload-Protocol" +HEADER_COMMAND = "X-Goog-Upload-Command" +HEADER_STATUS = "X-Goog-Upload-Status" +HEADER_URL = "X-Goog-Upload-URL" +HEADER_OFFSET = "X-Goog-Upload-Offset" +HEADER_SIZE_RECEIVED = "X-Goog-Upload-Size-Received" +HEADER_CONTENT_TYPE = "X-Goog-Upload-Header-Content-Type" +HEADER_CONTENT_LENGTH = "X-Goog-Upload-Header-Content-Length" +HEADER_CHUNK_GRANULARITY = "X-Goog-Upload-Chunk-Granularity" + +PROTOCOL_RESUMABLE = "resumable" + + +class Command(str, enum.Enum): + """Protocol commands.""" + + START = "start" + UPLOAD = "upload" + FINALIZE = "finalize" + QUERY = "query" + CANCEL = "cancel" + + +class Status(str, enum.Enum): + """Server upload status values.""" + + ACTIVE = "active" + FINAL = "final" + CANCELLED = "cancelled" + + +class ProgressState(str, enum.Enum): + """Progress notification state values.""" + + STARTED = "started" + UPLOADING = "uploading" + RECOVERING = "recovering" + OFFSET_RECEIVED = "offset received" + FINALIZED = "finalized" + + +@dataclasses.dataclass(frozen=True) +class UploadProgress: + """Upload progress notification payload. + + Attributes: + upload_url: The unique session URL for this upload. + chunk_size: The actual negotiated chunk size. + bytes_uploaded: The total confirmed bytes committed so far. + total_bytes: The total size of the stream in bytes, if known. + state: The current progress state. + """ + + upload_url: str + chunk_size: int + bytes_uploaded: int + total_bytes: Optional[int] + state: ProgressState + + +# HTTP status codes indicating transient retryable errors +RETRYABLE_STATUS_CODES = (408, 429, 500, 502, 503, 504) + +# HTTP status codes indicating state consistency errors requiring recovery +RECOVERABLE_STATUS_CODES = (400, 409, 412, 416) + diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py new file mode 100644 index 000000000000..57caaccafc71 --- /dev/null +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -0,0 +1,907 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Synchronous Resumable Upload session and helpers using requests.""" + +import contextlib +import dataclasses +import datetime +import io +import logging +import time +from typing import ( + Any, + BinaryIO, + Callable, + Generator, + Iterable, + List, + Mapping, + Optional, + Sequence, + Tuple, + Union, +) + +import requests + +import google.api_core.retry +import google.protobuf.message +import proto +from google.api_core import exceptions +from google.api_core.resumable_transfer import common, upload_state +from google.protobuf import json_format + +_LOGGER = logging.getLogger(__name__) +_DEFAULT_START_TIMEOUT = 60.0 # seconds for initial start request +_monotonic_clock = time.monotonic + + +class _RecoveryRetransmit(Exception): + """Internal exception indicating state synchronization succeeded and chunk should retransmit.""" + + pass + + +@dataclasses.dataclass +class ResumableUploadConfig: + """Configuration options for a resumable upload. + + Attributes: + chunk_size: Size in bytes for each uploaded data chunk. Defaults to 10 MiB. + start_timeout: Local per-request timeout in seconds for start request. + start_retry: Custom retry policy for the start request. + stall_minimum_rate: Minimum transfer rate in bytes per second. Defaults to 64 KiB/s. + stall_timeout: Stall duration threshold in seconds. Defaults to 120s. + additional_headers: Additional HTTP headers dispatched exclusively with start request. + deadline: Overall global deadline for the upload process. + timeout: Fallback per-request timeout. + retry: Fallback retry policy. + on_progress: Callback function receiving UploadProgress notifications. + response_type: Optional message class (proto.Message or google.protobuf.message.Message), + callable deserializer, or None to return raw response. + content_type: MIME type of the stream payload. + """ + + chunk_size: int = common.DEFAULT_CHUNK_SIZE + start_timeout: Optional[float] = None + start_retry: Optional[google.api_core.retry.Retry] = None + stall_minimum_rate: int = 64 * 1024 + stall_timeout: float = 120.0 + additional_headers: Optional[ + Union[Mapping[str, str], Sequence[Tuple[str, str]]] + ] = None + deadline: Optional[datetime.datetime] = None + timeout: Optional[float] = None + retry: Optional[google.api_core.retry.Retry] = None + on_progress: Optional[Callable[[common.UploadProgress], None]] = None + response_type: Optional[Any] = None + content_type: Optional[str] = None + + def __post_init__(self) -> None: + """Normalizes fallback timeouts and retry policies.""" + if self.start_timeout is not None and self.timeout is None: + self.timeout = self.start_timeout + elif self.timeout is not None and self.start_timeout is None: + self.start_timeout = self.timeout + + if self.start_retry is not None and self.retry is None: + self.retry = self.start_retry + elif self.retry is not None and self.start_retry is None: + self.start_retry = self.retry + + @property + def start_headers(self) -> Optional[Sequence[Tuple[str, str]]]: + """Returns normalized additional headers for the start request.""" + if self.additional_headers is None: + return None + if isinstance(self.additional_headers, Mapping): + return list(self.additional_headers.items()) + return list(self.additional_headers) + + +class ResumableUploadSession: + """Manages the full lifecycle of a resumable upload session.""" + + def __init__( + self, + upload_url: Optional[str] = None, + config: Optional[ResumableUploadConfig] = None, + resumable_url: Optional[str] = None, + transport: Optional[requests.Session] = None, + ) -> None: + """Initializes a ResumableUploadSession. + + Args: + upload_url: The initial URL for the start request. + config: Optional upload configuration parameters. + resumable_url: Pre-existing upload session URL if resuming. + transport: Optional requests session. + """ + self._config = config or ResumableUploadConfig() + self._transport = transport + self._response: Optional[Any] = None + self._state = upload_state.ProtocolState( + upload_url=upload_url, + chunk_size=self._config.chunk_size, + resumable_url=resumable_url, + ) + + # In-memory zero-copy buffer (never discard chunk until confirmed) + self._buffered_chunk: Optional[memoryview] = None + self._buffered_chunk_offset: int = 0 + self._start_stream_offset: int = 0 + + # Stall control tracking via monotonic clock + self._aggregate_lag: float = 0.0 + self._stall_timeout_started: Optional[float] = None + + @property + def upload_url(self) -> Optional[str]: + """Optional[str]: The unique upload URL for this session.""" + return self._state.resumable_url + + @property + def chunk_size(self) -> int: + """int: The negotiated chunk size.""" + return self._state.chunk_size + + @property + def response(self) -> Optional[Any]: + """Optional[Any]: The cached response message if finished.""" + return self._response + + @property + def bytes_uploaded(self) -> int: + """int: Confirmed number of bytes committed so far.""" + return self._state.bytes_uploaded + + @property + def finished(self) -> bool: + """bool: Whether the upload has completed successfully.""" + return self._state.finished + + def _get_transport( + self, transport: Optional[requests.Session] + ) -> requests.Session: + """Resolves the requests.Session transport. + + Args: + transport: Explicit requests session if provided. + + Returns: + The resolved requests session. + + Raises: + ValueError: If no requests session is available. + """ + sess = transport or self._transport + if sess is None: + raise ValueError("A requests.Session transport must be provided.") + return sess + + def _enrich_exception(self, exc: BaseException) -> None: + """Attaches session diagnostic metadata to an active exception. + + Args: + exc: Exception instance to augment with upload_url and chunk_size. + """ + if hasattr(exc, "__dict__"): + exc.upload_url = self.upload_url + exc.chunk_size = self.chunk_size + + def _notify_progress(self, state: common.ProgressState) -> None: + """Notifies registered progress callback with current upload status. + + Args: + state: ProgressState transition milestone. + """ + if self._config.on_progress and self.upload_url: + self._config.on_progress( + common.UploadProgress( + upload_url=self.upload_url, + chunk_size=self.chunk_size, + bytes_uploaded=self._state.bytes_uploaded, + total_bytes=self._state.total_bytes, + state=state, + ) + ) + + @contextlib.contextmanager + def _capture_progress( + self, + ) -> Generator[List[common.UploadProgress], None, None]: + """Intercepts progress events to buffer snapshots for generator consumers. + + Yields: + List buffering UploadProgress snapshots during generator execution. + """ + captured: List[common.UploadProgress] = [] + old_cb = self._config.on_progress + + def capture(p: common.UploadProgress) -> None: + captured.append(p) + if old_cb: + old_cb(p) + + self._config.on_progress = capture + try: + yield captured + finally: + self._config.on_progress = old_cb + + def _get_deadline_remaining(self) -> Optional[float]: + """Calculates remaining seconds until the configured upload deadline. + + Returns: + Remaining seconds before deadline, or None if no deadline configured. + + Raises: + exceptions.DeadlineExceeded: If deadline has already elapsed. + """ + if self._config.deadline: + now = datetime.datetime.now(datetime.timezone.utc) + dl = self._config.deadline + if dl.tzinfo is None: + dl = dl.replace(tzinfo=datetime.timezone.utc) + remaining = (dl - now).total_seconds() + if remaining <= 0: + raise exceptions.DeadlineExceeded( + f"Resumable upload deadline {self._config.deadline} exceeded." + ) + return remaining + return None + + def _get_start_timeout(self) -> float: + """Computes timeout in seconds for start and control requests. + + Returns: + Applicable timeout in seconds. + """ + remaining = self._get_deadline_remaining() + timeout = ( + self._config.start_timeout or self._config.timeout or _DEFAULT_START_TIMEOUT + ) + if remaining is not None: + return min(timeout, remaining) + return timeout + + def _get_retry_predicate(self) -> Callable[[Any], bool]: + """Returns a predicate function for determining if an exception is retryable. + + Returns: + A callable accepting an exception and returning a boolean. + """ + def should_retry(exc: Any) -> bool: + if isinstance( + exc, + ( + exceptions.DeadlineExceeded, + exceptions.TransferStalledError, + exceptions.UploadCancelledError, + ), + ): + return False + if isinstance(exc, exceptions.MissingStatusHeaderError): + return True + if isinstance(exc, requests.exceptions.RequestException): + if isinstance( + exc, + ( + requests.exceptions.ConnectionError, + requests.exceptions.ChunkedEncodingError, + ), + ): + return True + if isinstance(exc, requests.exceptions.Timeout): + if self._config.stall_minimum_rate and self._config.stall_timeout: + return False + return True + if isinstance(exc, exceptions.GoogleAPICallError): + return exc.code in common.RETRYABLE_STATUS_CODES + return False + + return should_retry + + def _get_retry(self, is_start: bool = False) -> google.api_core.retry.Retry: + """Resolves retry policy for requests. + + Args: + is_start: Whether this retry policy is for the start request. + + Returns: + Configured or default Retry instance. + """ + if is_start and self._config.start_retry: + return self._config.start_retry + if self._config.retry: + return self._config.retry + return google.api_core.retry.Retry(predicate=self._get_retry_predicate()) + + def _compute_chunk_timeout(self, data_len: int) -> float: + """Computes the dynamic per-attempt chunk timeout based on stall control and deadlines. + + Args: + data_len: Length of the current chunk in bytes. + + Returns: + Timeout in seconds for chunk transmission attempt. + """ + rate = self._config.stall_minimum_rate + expected_sec = data_len / rate if rate > 0 else 60.0 + next_chunk_timeout = max( + 1.0, + expected_sec - self._aggregate_lag + self._config.stall_timeout, + ) + per_attempt_timeout = max(5.0, min(next_chunk_timeout, 2.0 * expected_sec)) + + if self._config.timeout: + per_attempt_timeout = min(self._config.timeout, per_attempt_timeout) + + remaining = self._get_deadline_remaining() + if remaining is not None: + per_attempt_timeout = min(per_attempt_timeout, remaining) + + return per_attempt_timeout + + def _update_stall_control( + self, data_len: int, t_start: float, t_elapsed: float + ) -> None: + """Updates aggregate transfer rate lag and enforces stall timeout and deadlines. + + Args: + data_len: Length of the transmitted chunk in bytes. + t_start: Monotonic timestamp before chunk transmission began. + t_elapsed: Elapsed duration in seconds for chunk transmission. + + Raises: + exceptions.DeadlineExceeded: If upload deadline is exceeded. + exceptions.TransferStalledError: If transfer throughput stalls past configured timeout. + """ + if not (self._config.stall_minimum_rate and self._config.stall_timeout): + return + + rate = self._config.stall_minimum_rate + expected_sec = data_len / rate if rate > 0 else 0.0 + current_lag = t_elapsed - expected_sec + self._aggregate_lag = max(0.0, self._aggregate_lag + current_lag) + + if self._aggregate_lag > 0.0: + if self._stall_timeout_started is None: + self._stall_timeout_started = t_start + if ( + _monotonic_clock() - self._stall_timeout_started + >= self._config.stall_timeout + ): + remaining = self._get_deadline_remaining() + if remaining is not None and remaining <= 0: + raise exceptions.DeadlineExceeded( + f"Resumable upload deadline {self._config.deadline} exceeded." + ) + raise exceptions.TransferStalledError( + f"Upload stalled: transfer rate remained below {rate} bytes/s " + f"for longer than {self._config.stall_timeout}s." + ) + else: + self._stall_timeout_started = None + + def _reposition_stream_offset( + self, stream: BinaryIO, received: int + ) -> int: + """Adjusts in-memory chunk buffer or seeks input stream to server offset. + + Args: + stream: The input data stream. + received: Confirmed byte offset committed on the server. + + Returns: + The confirmed server byte offset. + + Raises: + exceptions.UnseekableStreamError: If server offset precedes buffer and stream cannot be rewound. + """ + if self._buffered_chunk is not None: + chunk_start = self._buffered_chunk_offset + chunk_end = chunk_start + len(self._buffered_chunk) + if chunk_start <= received <= chunk_end: + discard_len = received - chunk_start + self._buffered_chunk = self._buffered_chunk[discard_len:] + self._buffered_chunk_offset = received + return received + + self._buffered_chunk = None + if hasattr(stream, "seekable") and not stream.seekable(): + err = exceptions.UnseekableStreamError( + f"Stream is not seekable. Cannot recover upload to offset {received}." + ) + self._enrich_exception(err) + raise err + try: + stream.seek(self._start_stream_offset + received) + except (OSError, AttributeError) as exc: + err = exceptions.UnseekableStreamError( + f"Failed to seek stream to offset {received}: {exc}" + ) + self._enrich_exception(err) + raise err from exc + + return received + + def initiate( + self, + transport: requests.Session, + request_body: Union[str, bytes] = "", + size: Optional[int] = None, + ) -> str: + """Initiates the upload session by sending the start command. + + Args: + transport: The requests session. + request_body: JSON payload for initial start request. + size: Total size of payload in bytes, if known. + + Returns: + The upload session URL. + """ + method, url, headers, payload = self._state.build_start_request( + body=request_body, + headers=self._config.start_headers, + content_type=self._config.content_type, + size=size, + ) + + def do_initiate() -> str: + timeout = self._get_start_timeout() + response = transport.request( + method, url, data=payload, headers=headers, timeout=timeout + ) + if not response.ok: + raise exceptions.from_http_response(response) + session_url = self._state.process_start_response( + response.status_code, response.headers + ) + return session_url + + session_url = self._get_retry(is_start=True)(do_initiate)() + self._notify_progress(common.ProgressState.STARTED) + return session_url + + def _transmit_chunk( + self, transport: requests.Session, stream: BinaryIO, size: Optional[int] + ) -> requests.Response: + """Transmits the next data chunk with stall control and error recovery. + + Args: + transport: The requests session. + stream: The input data stream. + size: Total size of the stream in bytes, if known. + + Returns: + The HTTP response for the transmitted chunk. + """ + def do_transmit() -> requests.Response: + chunk_size = self._state.chunk_size + + # Retain active chunk in zero-copy buffer if not present + if self._buffered_chunk is None: + raw_bytes = stream.read(chunk_size) + if not raw_bytes: + raw_bytes = b"" + self._buffered_chunk = memoryview(raw_bytes) + self._buffered_chunk_offset = self._state.bytes_uploaded + + data = self._buffered_chunk + data_len = len(data) + + is_last = data_len < chunk_size + if size is not None and self._state.bytes_uploaded + data_len >= size: + is_last = True + + method, url, headers, payload = self._state.build_chunk_request( + data=data, + is_last_chunk=is_last, + content_type=self._config.content_type, + ) + + def do_http() -> requests.Response: + per_attempt_timeout = self._compute_chunk_timeout(data_len) + resp = transport.request( + method, + url, + data=payload, + headers=headers, + timeout=per_attempt_timeout, + ) + if not resp.ok: + raise exceptions.from_http_response(resp) + return resp + + try: + t_start = _monotonic_clock() + resp = self._get_retry()(do_http)() + t_elapsed = _monotonic_clock() - t_start + + self._update_stall_control(data_len, t_start, t_elapsed) + self._state.process_chunk_response( + resp.status_code, resp.headers, data_len + ) + self._buffered_chunk = None + self._notify_progress( + common.ProgressState.FINALIZED + if self._state.finished + else common.ProgressState.UPLOADING + ) + return resp + except Exception as exc: + self._enrich_exception(exc) + if isinstance( + exc, (requests.exceptions.Timeout, exceptions.DeadlineExceeded) + ): + remaining = self._get_deadline_remaining() + if remaining is not None and remaining <= 0: + raise exceptions.DeadlineExceeded( + f"Resumable upload deadline {self._config.deadline} exceeded." + ) from exc + stalled_err = exceptions.TransferStalledError( + f"Upload stalled: chunk transfer timed out ({exc})." + ) + self._enrich_exception(stalled_err) + raise stalled_err from exc + + is_recoverable = ( + isinstance(exc, exceptions.GoogleAPICallError) + and exc.code in common.RECOVERABLE_STATUS_CODES + ) or isinstance(exc, exceptions.MissingStatusHeaderError) + + if is_recoverable: + _LOGGER.info( + "Recoverable error %s during chunk upload. Querying server offset.", + exc, + ) + self._notify_progress(common.ProgressState.RECOVERING) + self._recover(transport, stream) + raise _RecoveryRetransmit() + raise + + recovery_loop = google.api_core.retry.Retry( + predicate=lambda e: isinstance(e, _RecoveryRetransmit) + ) + return recovery_loop(do_transmit)() + + def _recover(self, transport: requests.Session, stream: BinaryIO) -> int: + """Queries server for committed byte offset and adjusts buffer / stream. + + Args: + transport: The requests session. + stream: The input data stream. + + Returns: + The confirmed server byte offset. + + Raises: + exceptions.UnseekableStreamError: If server offset precedes buffer and stream cannot be rewound. + exceptions.GoogleAPICallError: If query request fails on the server. + """ + method, url, headers, payload = self._state.build_query_request() + + def do_query() -> requests.Response: + timeout = self._get_start_timeout() + resp = transport.request( + method, url, data=payload, headers=headers, timeout=timeout + ) + if not resp.ok: + raise exceptions.from_http_response(resp) + return resp + + resp = self._get_retry()(do_query)() + received = self._state.process_query_response(resp.status_code, resp.headers) + self._notify_progress(common.ProgressState.OFFSET_RECEIVED) + return self._reposition_stream_offset(stream, received) + + def cancel(self, transport: Optional[requests.Session] = None) -> None: + """Cancels the resumable upload session. + + Args: + transport: Optional requests session to use for dispatching cancellation. + + Raises: + ValueError: If no requests session is available. + GoogleAPICallError: If the cancellation request fails on the server. + """ + sess = self._get_transport(transport) + method, url, headers, payload = self._state.build_cancel_request() + timeout = self._get_start_timeout() + resp = sess.request(method, url, data=payload, headers=headers, timeout=timeout) + if not resp.ok: + raise exceptions.from_http_response(resp) + self._state.process_cancel_response(resp.status_code, resp.headers) + + def _transmit_all_chunks( + self, + transport: requests.Session, + stream_obj: BinaryIO, + computed_size: Optional[int], + captured: Optional[List[common.UploadProgress]] = None, + ) -> Generator[common.UploadProgress, None, None]: + """Transmits chunks until transfer completes, yielding buffered progress updates. + + Args: + transport: The requests session. + stream_obj: Binary stream yielding upload chunks. + computed_size: Total payload size in bytes if known. + captured: Optional buffer accumulating progress snapshots. + + Yields: + UploadProgress snapshots for each transmission milestone. + + Raises: + ValueError: If upload concludes without a server response. + """ + if captured: + while captured: + yield captured.pop(0) + + final_resp = None + while not self._state.finished and not self._state.invalid: + final_resp = self._transmit_chunk(transport, stream_obj, computed_size) + if captured: + while captured: + yield captured.pop(0) + + if final_resp is None: + raise ValueError("Upload completed without receiving a final response.") + + self._response = self._format_response(final_resp) + + def upload( + self, + stream: Union[BinaryIO, bytes, Iterable[bytes]], + request_body: Union[str, bytes] = "", + size: Optional[int] = None, + transport: Optional[requests.Session] = None, + ) -> Any: + """Executes the resumable upload from start to completion. + + Args: + stream: Data payload to upload (file-like stream, bytes, or iterable of bytes). + request_body: Initial metadata payload sent with the start request. + size: Total stream size in bytes, if known. + transport: Optional requests session. + + Returns: + The final server response payload or deserialized response message. + + Raises: + ValueError: If transport is missing or upload completes without a response. + GoogleAPICallError: If an unrecoverable API error occurs. + """ + for _ in self.iter_upload( + stream=stream, request_body=request_body, size=size, transport=transport + ): + pass + return self._response + + def iter_upload( + self, + stream: Union[BinaryIO, bytes, Iterable[bytes]], + request_body: Union[str, bytes] = "", + size: Optional[int] = None, + transport: Optional[requests.Session] = None, + ) -> Generator[common.UploadProgress, None, None]: + """Streams upload execution, yielding UploadProgress snapshots (PEP 255). + + Args: + stream: Data payload to upload (file-like stream, bytes, or iterable of bytes). + request_body: Initial metadata payload sent with the start request. + size: Total stream size in bytes, if known. + transport: Optional requests session. + + Yields: + UploadProgress snapshots for each chunk transmission milestone. + + Raises: + ValueError: If transport is missing or upload completes without a response. + GoogleAPICallError: If an unrecoverable API error occurs. + """ + sess = self._get_transport(transport) + with self._capture_progress() as captured: + try: + stream_obj, computed_size = self._prepare_stream(stream, size) + self.initiate( + transport=sess, request_body=request_body, size=computed_size + ) + yield from self._transmit_all_chunks( + sess, stream_obj, computed_size, captured + ) + except Exception as exc: + self._enrich_exception(exc) + raise + + def resume( + self, + upload_url: Optional[str] = None, + stream: Optional[Union[BinaryIO, bytes, Iterable[bytes]]] = None, + size: Optional[int] = None, + chunk_size: Optional[int] = None, + transport: Optional[requests.Session] = None, + ) -> Any: + """Resumes an existing upload from a saved upload URL. + + Args: + upload_url: The pre-existing upload session URL. + stream: The data payload to resume uploading from. + size: Total size of the payload in bytes, if known. + chunk_size: Optional chunk size override in bytes. + transport: Optional requests session. + + Returns: + The final server response payload or deserialized response message. + + Raises: + ValueError: If required arguments are missing or response not received. + GoogleAPICallError: If an unrecoverable API error occurs. + """ + for _ in self.iter_resume( + upload_url=upload_url, + stream=stream, + size=size, + chunk_size=chunk_size, + transport=transport, + ): + pass + return self._response + + def iter_resume( + self, + upload_url: Optional[str] = None, + stream: Optional[Union[BinaryIO, bytes, Iterable[bytes]]] = None, + size: Optional[int] = None, + chunk_size: Optional[int] = None, + transport: Optional[requests.Session] = None, + ) -> Generator[common.UploadProgress, None, None]: + """Streams resumption of an upload, yielding UploadProgress snapshots. + + Args: + upload_url: The pre-existing upload session URL. + stream: The data payload to resume uploading from. + size: Total size of the payload in bytes, if known. + chunk_size: Optional chunk size override in bytes. + transport: Optional requests session. + + Yields: + UploadProgress snapshots for each chunk transmission milestone. + + Raises: + ValueError: If required arguments are missing or response not received. + GoogleAPICallError: If an unrecoverable API error occurs. + """ + sess = self._get_transport(transport) + actual_url = upload_url or self.upload_url + if not actual_url: + raise ValueError("An upload URL must be provided to resume.") + if stream is None: + raise ValueError("A data stream or payload must be provided to resume.") + + if chunk_size is not None: + self._state._chunk_size = chunk_size + + self._state._resumable_url = actual_url + with self._capture_progress() as captured: + try: + stream_obj, computed_size = self._prepare_stream(stream, size) + self._recover(sess, stream_obj) + yield from self._transmit_all_chunks( + sess, stream_obj, computed_size, captured + ) + except Exception as exc: + self._enrich_exception(exc) + raise + + def _prepare_stream( + self, stream: Union[BinaryIO, bytes, Iterable[bytes]], size: Optional[int] + ) -> Tuple[BinaryIO, Optional[int]]: + """Normalizes stream input into a BinaryIO object and determines stream length. + + Args: + stream: Input stream, bytes, or iterable of bytes. + size: Explicit total size in bytes, if known. + + Returns: + Tuple of (prepared BinaryIO stream, computed total size). + """ + computed_size = size + if isinstance(stream, bytes): + stream_obj: BinaryIO = io.BytesIO(stream) + if computed_size is None: + computed_size = len(stream) + elif not hasattr(stream, "read") and isinstance(stream, Iterable): + stream_obj = io.BytesIO(b"".join(stream)) + if computed_size is None: + computed_size = stream_obj.getbuffer().nbytes + else: + stream_obj = stream + if computed_size is None: + if hasattr(stream_obj, "getbuffer"): + computed_size = stream_obj.getbuffer().nbytes + elif ( + hasattr(stream_obj, "seekable") + and stream_obj.seekable() + and hasattr(stream_obj, "tell") + ): + cur = stream_obj.tell() + stream_obj.seek(0, io.SEEK_END) + computed_size = stream_obj.tell() - cur + stream_obj.seek(cur) + + if hasattr(stream_obj, "tell"): + try: + self._start_stream_offset = stream_obj.tell() + except (OSError, AttributeError): + self._start_stream_offset = 0 + + return stream_obj, computed_size + + def _format_response(self, response: requests.Response) -> Any: + """Formats response into protobuf message type if provided. + + Args: + response: HTTP response object from final chunk. + + Returns: + Deserialized protobuf message or the raw response object. + """ + return _format_response_payload(response, self._config.response_type) + + +def _format_response_payload( + response: Union[Any, bytes], + response_type: Optional[Any], +) -> Any: + """Formats raw response or bytes into protobuf or proto-plus message type if configured. + + Args: + response: Raw HTTP response object or response body bytes. + response_type: Deserializer callable, proto.Message class, or + google.protobuf.message.Message class or instance. + + Returns: + Deserialized protobuf message or the raw response object / bytes. + """ + if response_type is None: + return response + + content: bytes + if isinstance(response, bytes): + content = response + elif hasattr(response, "content"): + content = response.content + else: + content = bytes(response) + + if isinstance(response_type, type) and issubclass(response_type, proto.Message): + return response_type.from_json(content, ignore_unknown_fields=True) + if isinstance(response_type, type) and issubclass( + response_type, google.protobuf.message.Message + ): + instance = response_type() + return json_format.Parse(content, instance, ignore_unknown_fields=True) + if isinstance(response_type, google.protobuf.message.Message): + return json_format.Parse(content, response_type, ignore_unknown_fields=True) + if hasattr(response_type, "from_json") and callable(response_type.from_json): + return response_type.from_json(content) + if callable(response_type): + return response_type(content) + + return response + diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py new file mode 100644 index 000000000000..2a1c80dfade9 --- /dev/null +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -0,0 +1,854 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Asynchronous Resumable Upload session and helpers using aiohttp.""" + +import asyncio +import datetime +import inspect +import io +import logging +import time +from typing import ( + Any, + AsyncIterable, + AsyncIterator, + Awaitable, + BinaryIO, + Callable, + Generator, + Generic, + Iterable, + Mapping, + Optional, + Tuple, + TypeVar, + Union, +) + +try: + import aiohttp +except ImportError: # pragma: NO COVER + aiohttp = None # type: ignore + +from google.api_core import exceptions +from google.api_core.resumable_transfer import common, upload_state +from google.api_core.resumable_transfer.upload import ( + ResumableUploadConfig, + _format_response_payload, +) + +_LOGGER = logging.getLogger(__name__) +_DEFAULT_START_TIMEOUT = 60.0 # seconds for initial start request +_DONE_SENTINEL = object() +_monotonic_clock = time.monotonic + +ResponseProto = TypeVar("ResponseProto") + + +class _AsyncRecoveryRetransmit(Exception): + """Internal exception indicating state synchronization succeeded and chunk should retransmit.""" + + pass + + +class AsyncUploadOperation(Generic[ResponseProto], Awaitable[ResponseProto]): + """Handle representing an active asynchronous upload operation. + + Implements Awaitable[ResponseProto] so awaiting the operation directly + returns the deserialized response upon transfer completion. + """ + + def __init__( + self, + task: asyncio.Task, + session: "AsyncResumableUploadSession", + progress_queue: asyncio.Queue, + ) -> None: + """Initializes the active upload operation handle. + + Args: + task: Background asyncio task driving the upload. + session: Underlying asynchronous resumable upload session. + progress_queue: Queue used to deliver upload progress updates. + """ + self._task = task + self._session = session + self._progress_queue = progress_queue + + def __await__(self) -> Generator[Any, None, ResponseProto]: + """Awaits completion of the upload task and returns the server response.""" + return self._task.__await__() + + async def progress(self) -> AsyncIterator[common.UploadProgress]: + """Returns an asynchronous stream yielding progress snapshots without blocking uploads. + + Yields: + UploadProgress snapshots for each progress transition. + + Raises: + Exception: Re-raises any exception encountered during the background transfer. + """ + while True: + item = await self._progress_queue.get() + if item is _DONE_SENTINEL: + break + if isinstance(item, Exception): + raise item + yield item + + @property + def response(self) -> Optional[ResponseProto]: + """The deserialized protobuf response message, or None if in progress.""" + return self._session.response + + @property + def upload_url(self) -> Optional[str]: + """The session upload URL.""" + return self._session.upload_url + + @property + def chunk_size(self) -> int: + """The negotiated chunk size.""" + return self._session.chunk_size + + @property + def bytes_uploaded(self) -> int: + """Total confirmed bytes committed so far.""" + return self._session.bytes_uploaded + + +class AsyncResumableUploadSession: + """Manages the full lifecycle of an asynchronous resumable upload session.""" + + def __init__( + self, + upload_url: Optional[str] = None, + config: Optional[ResumableUploadConfig] = None, + resumable_url: Optional[str] = None, + transport: Optional[Any] = None, + ) -> None: + """Initializes an AsyncResumableUploadSession. + + Args: + upload_url: The initial URL for the start request. + config: Optional upload configuration parameters. + resumable_url: Pre-existing upload session URL if resuming. + transport: Optional aiohttp.ClientSession. + """ + self._config = config or ResumableUploadConfig() + self._transport = transport + self._response: Optional[Any] = None + self._state = upload_state.ProtocolState( + upload_url=upload_url, + chunk_size=self._config.chunk_size, + resumable_url=resumable_url, + ) + + # In-memory zero-copy buffer + self._buffered_chunk: Optional[memoryview] = None + self._buffered_chunk_offset: int = 0 + self._start_stream_offset: int = 0 + + # Stall control tracking via monotonic clock + self._aggregate_lag: float = 0.0 + self._stall_timeout_started: Optional[float] = None + + @property + def upload_url(self) -> Optional[str]: + """Optional[str]: The unique upload URL for this session.""" + return self._state.resumable_url + + @property + def chunk_size(self) -> int: + """int: The negotiated chunk size.""" + return self._state.chunk_size + + @property + def response(self) -> Optional[Any]: + """Optional[Any]: The cached response message if finished.""" + return self._response + + @property + def bytes_uploaded(self) -> int: + """int: Confirmed number of bytes committed so far.""" + return self._state.bytes_uploaded + + @property + def finished(self) -> bool: + """bool: Whether the upload has completed successfully.""" + return self._state.finished + + def _ensure_aiohttp(self) -> None: + """Validates that aiohttp is installed and accessible. + + Raises: + ImportError: If aiohttp is not installed. + """ + if aiohttp is None: + raise ImportError( + "The aiohttp library is required to use AsyncResumableUploadSession. " + "Please install google-api-core[async_rest]." + ) + + def _notify_progress( + self, state: common.ProgressState, queue: Optional[asyncio.Queue] = None + ) -> None: + """Notifies registered progress callback and queue with current upload status. + + Args: + state: ProgressState transition milestone. + queue: Optional queue to receive progress event. + """ + if self.upload_url: + progress = common.UploadProgress( + upload_url=self.upload_url, + chunk_size=self.chunk_size, + bytes_uploaded=self._state.bytes_uploaded, + total_bytes=self._state.total_bytes, + state=state, + ) + if self._config.on_progress: + try: + self._config.on_progress(progress) + except Exception: # pragma: NO COVER + pass + if queue is not None: + queue.put_nowait(progress) + + def _get_deadline_remaining(self) -> Optional[float]: + """Calculates remaining seconds until the configured upload deadline. + + Returns: + Remaining seconds before deadline, or None if no deadline configured. + + Raises: + exceptions.DeadlineExceeded: If deadline has already elapsed. + """ + if self._config.deadline: + now = datetime.datetime.now(datetime.timezone.utc) + dl = self._config.deadline + if dl.tzinfo is None: + dl = dl.replace(tzinfo=datetime.timezone.utc) + remaining = (dl - now).total_seconds() + if remaining <= 0: + raise exceptions.DeadlineExceeded( + f"Resumable upload deadline {self._config.deadline} exceeded." + ) + return remaining + return None + + def _get_start_timeout(self) -> float: + """Computes timeout in seconds for start and control requests. + + Returns: + Applicable timeout in seconds. + """ + remaining = self._get_deadline_remaining() + timeout = ( + self._config.start_timeout or self._config.timeout or _DEFAULT_START_TIMEOUT + ) + if remaining is not None: + return min(timeout, remaining) + return timeout + + async def _async_retry( + self, coro_fn: Callable[[], Awaitable[Any]], max_attempts: int = 4 + ) -> Any: + """Executes an asynchronous callable with exponential backoff retry logic. + + Args: + coro_fn: Asynchronous nullary function to invoke and retry. + max_attempts: Maximum retry attempts before propagating failure. + + Returns: + The successful return value of coro_fn. + """ + delay = 1.0 + multiplier = 2.0 + max_delay = 60.0 + for attempt in range(max_attempts): + try: + return await coro_fn() + except ( + exceptions.DeadlineExceeded, + exceptions.TransferStalledError, + exceptions.UploadCancelledError, + ): + raise + except exceptions.MissingStatusHeaderError: + if attempt == max_attempts - 1: + raise + except exceptions.GoogleAPICallError as exc: + if exc.code not in common.RETRYABLE_STATUS_CODES: + raise + if attempt == max_attempts - 1: + raise + except Exception: + if attempt == max_attempts - 1: + raise + + await asyncio.sleep(delay) + delay = min(delay * multiplier, max_delay) + + async def initiate( + self, + transport: Any, + request_body: Union[str, bytes] = "", + size: Optional[int] = None, + progress_queue: Optional[asyncio.Queue] = None, + ) -> str: + """Initiates the upload session by sending the start command asynchronously. + + Args: + transport: The aiohttp client session. + request_body: Initial metadata payload sent with start command. + size: Total stream size in bytes, if known. + progress_queue: Optional queue to receive progress event. + + Returns: + The upload session URL. + + Raises: + GoogleAPICallError: If the server rejects the start request. + MissingStatusHeaderError: If the server response lacks status header. + """ + self._ensure_aiohttp() + method, url, headers, payload = self._state.build_start_request( + body=request_body, + headers=self._config.start_headers, + content_type=self._config.content_type, + size=size, + ) + + async def do_initiate(): + timeout_sec = self._get_start_timeout() + client_timeout = aiohttp.ClientTimeout(total=timeout_sec) + async with transport.request( + method, url, data=payload, headers=headers, timeout=client_timeout + ) as resp: + resp_headers = dict(resp.headers) + body = await resp.read() + if resp.status not in (200, 201): + raise exceptions.from_http_status( + resp.status, body.decode("utf-8", errors="replace") + ) + session_url = self._state.process_start_response( + resp.status, resp_headers + ) + return session_url + + session_url = await self._async_retry(do_initiate) + self._notify_progress(common.ProgressState.STARTED, progress_queue) + return session_url + + async def _transmit_chunk( + self, + transport: Any, + reader_fn: Callable[[int], Awaitable[bytes]], + size: Optional[int], + progress_queue: Optional[asyncio.Queue] = None, + stream_obj: Any = None, + ) -> Tuple[int, Mapping[str, str], bytes]: + """Transmits the next data chunk asynchronously with stall control. + + Args: + transport: The aiohttp client session. + reader_fn: Async callable returning chunk bytes. + size: Total stream size in bytes, if known. + progress_queue: Optional queue to receive progress updates. + stream_obj: Underlying stream object for recovery seeking. + + Returns: + Tuple of (status code, headers mapping, response body bytes). + + Raises: + TransferStalledError: If chunk transfer throughput stalls. + DeadlineExceeded: If upload deadline is reached. + GoogleAPICallError: If chunk upload encounters an unrecoverable error. + """ + async def do_transmit(): + chunk_size = self._state.chunk_size + + # Retain active chunk in zero-copy buffer if not present + if self._buffered_chunk is None: + raw_bytes = await reader_fn(chunk_size) + if not raw_bytes: + raw_bytes = b"" + self._buffered_chunk = memoryview(raw_bytes) + self._buffered_chunk_offset = self._state.bytes_uploaded + + data = self._buffered_chunk + data_len = len(data) + + is_last = data_len < chunk_size + if size is not None and self._state.bytes_uploaded + data_len >= size: + is_last = True + + method, url, headers, payload = self._state.build_chunk_request( + data=data, + is_last_chunk=is_last, + content_type=self._config.content_type, + ) + + async def do_http(): + rate = self._config.stall_minimum_rate + expected_sec = data_len / rate if rate > 0 else 60.0 + next_chunk_timeout = max( + 1.0, + expected_sec - self._aggregate_lag + self._config.stall_timeout, + ) + per_attempt_timeout = max( + 5.0, min(next_chunk_timeout, 2.0 * expected_sec) + ) + + if self._config.timeout: + per_attempt_timeout = min(self._config.timeout, per_attempt_timeout) + + remaining = self._get_deadline_remaining() + if remaining is not None: + per_attempt_timeout = min(per_attempt_timeout, remaining) + + client_timeout = aiohttp.ClientTimeout(total=per_attempt_timeout) + async with transport.request( + method, url, data=payload, headers=headers, timeout=client_timeout + ) as resp: + resp_headers = dict(resp.headers) + resp_body = await resp.read() + if resp.status not in (200, 201): + raise exceptions.from_http_status( + resp.status, resp_body.decode("utf-8", errors="replace") + ) + return resp.status, resp_headers, resp_body + + try: + t_start = _monotonic_clock() + status_code, resp_headers, resp_body = await self._async_retry(do_http) + t_elapsed = _monotonic_clock() - t_start + + # Evaluate stall control lag & timer + if self._config.stall_minimum_rate and self._config.stall_timeout: + rate = self._config.stall_minimum_rate + expected_sec = data_len / rate if rate > 0 else 0.0 + current_lag = t_elapsed - expected_sec + self._aggregate_lag = max(0.0, self._aggregate_lag + current_lag) + if self._aggregate_lag > 0.0: + if self._stall_timeout_started is None: + self._stall_timeout_started = t_start + if ( + _monotonic_clock() - self._stall_timeout_started + >= self._config.stall_timeout + ): + remaining = self._get_deadline_remaining() + if remaining is not None and remaining <= 0: + raise exceptions.DeadlineExceeded( + f"Resumable upload deadline {self._config.deadline} exceeded." + ) + raise exceptions.TransferStalledError( + f"Upload stalled: transfer rate remained below {rate} bytes/s for longer than {self._config.stall_timeout}s." + ) + else: + self._stall_timeout_started = None + + self._state.process_chunk_response(status_code, resp_headers, data_len) + self._buffered_chunk = None + self._notify_progress( + common.ProgressState.FINALIZED + if self._state.finished + else common.ProgressState.UPLOADING, + progress_queue, + ) + return status_code, resp_headers, resp_body + except Exception as exc: + if hasattr(exc, "__dict__"): + exc.upload_url = self.upload_url + exc.chunk_size = self.chunk_size + if isinstance(exc, (asyncio.TimeoutError, exceptions.DeadlineExceeded)): + remaining = self._get_deadline_remaining() + if remaining is not None and remaining <= 0: + raise exceptions.DeadlineExceeded( + f"Resumable upload deadline {self._config.deadline} exceeded." + ) from exc + stalled_err = exceptions.TransferStalledError( + f"Upload stalled: chunk transfer timed out ({exc})." + ) + stalled_err.upload_url = self.upload_url + stalled_err.chunk_size = self.chunk_size + raise stalled_err from exc + + is_recoverable = ( + isinstance(exc, exceptions.GoogleAPICallError) + and exc.code in common.RECOVERABLE_STATUS_CODES + ) or isinstance(exc, exceptions.MissingStatusHeaderError) + + if is_recoverable: + _LOGGER.info( + "Recoverable error %s during async chunk upload. Querying server offset.", + exc, + ) + self._notify_progress( + common.ProgressState.RECOVERING, progress_queue + ) + await self._recover(transport, stream_obj) + raise _AsyncRecoveryRetransmit() + raise + + while True: + try: + return await do_transmit() + except _AsyncRecoveryRetransmit: + continue + + async def _recover(self, transport: Any, stream_obj: Any = None) -> int: + """Queries server for committed byte offset and adjusts buffer. + + Args: + transport: The aiohttp client session. + stream_obj: Underlying stream object to rewind if seekable. + + Returns: + The confirmed server byte offset. + + Raises: + exceptions.UnseekableStreamError: If server offset precedes buffer and stream cannot be rewound. + exceptions.GoogleAPICallError: If query request fails on the server. + """ + method, url, headers, payload = self._state.build_query_request() + + async def do_query(): + timeout_sec = self._get_start_timeout() + client_timeout = aiohttp.ClientTimeout(total=timeout_sec) + async with transport.request( + method, url, data=payload, headers=headers, timeout=client_timeout + ) as resp: + resp_headers = dict(resp.headers) + body = await resp.read() + if resp.status not in (200, 201): + raise exceptions.from_http_status( + resp.status, body.decode("utf-8", errors="replace") + ) + return resp.status, resp_headers + + status_code, resp_headers = await self._async_retry(do_query) + received = self._state.process_query_response(status_code, resp_headers) + + if self._buffered_chunk is not None: + chunk_start = self._buffered_chunk_offset + chunk_end = chunk_start + len(self._buffered_chunk) + if chunk_start <= received <= chunk_end: + discard_len = received - chunk_start + self._buffered_chunk = self._buffered_chunk[discard_len:] + self._buffered_chunk_offset = received + return received + + self._buffered_chunk = None + if stream_obj is not None and hasattr(stream_obj, "seek"): + if hasattr(stream_obj, "seekable") and not stream_obj.seekable(): + err = exceptions.UnseekableStreamError( + f"Stream is not seekable. Cannot recover upload to offset {received}." + ) + err.upload_url = self.upload_url + err.chunk_size = self.chunk_size + raise err + try: + stream_obj.seek(self._start_stream_offset + received) + return received + except (OSError, AttributeError) as exc: + err = exceptions.UnseekableStreamError( + f"Failed to seek stream to offset {received}: {exc}" + ) + err.upload_url = self.upload_url + err.chunk_size = self.chunk_size + raise err from exc + + err = exceptions.UnseekableStreamError( + f"Server offset {received} precedes active buffer. Stream cannot be rewound." + ) + err.upload_url = self.upload_url + err.chunk_size = self.chunk_size + raise err + + async def cancel(self, transport: Optional[Any] = None) -> None: + """Cancels the resumable upload session asynchronously. + + Args: + transport: Optional aiohttp client session. + + Raises: + ValueError: If transport is missing. + exceptions.GoogleAPICallError: If cancellation request fails on the server. + """ + self._ensure_aiohttp() + sess = transport or self._transport + if sess is None: + raise ValueError("An aiohttp.ClientSession transport must be provided.") + method, url, headers, payload = self._state.build_cancel_request() + timeout_sec = self._get_start_timeout() + client_timeout = aiohttp.ClientTimeout(total=timeout_sec) + async with sess.request( + method, url, data=payload, headers=headers, timeout=client_timeout + ) as resp: + resp_headers = dict(resp.headers) + body = await resp.read() + if resp.status not in (200, 201): + raise exceptions.from_http_status( + resp.status, body.decode("utf-8", errors="replace") + ) + self._state.process_cancel_response(resp.status, resp_headers) + + def upload( + self, + stream: Union[AsyncIterable[bytes], BinaryIO, bytes, Iterable[bytes]], + request_body: Union[str, bytes] = "", + size: Optional[int] = None, + transport: Optional[Any] = None, + ) -> AsyncUploadOperation: + """Initiates and executes upload asynchronously, returning an AsyncUploadOperation. + + Args: + stream: Data payload to upload (async iterable, binary stream, bytes, or iterable). + request_body: Initial metadata payload sent with the start request. + size: Total stream size in bytes, if known. + transport: Optional aiohttp client session. + + Returns: + An AsyncUploadOperation handle representing the active transfer. + + Raises: + ValueError: If transport is missing. + """ + self._ensure_aiohttp() + sess = transport or self._transport + if sess is None: + raise ValueError("An aiohttp.ClientSession transport must be provided.") + + progress_queue: asyncio.Queue = asyncio.Queue() + + async def _run(): + try: + reader_fn, computed_size, stream_obj = self._prepare_async_reader( + stream, size + ) + await self.initiate( + transport=sess, + request_body=request_body, + size=computed_size, + progress_queue=progress_queue, + ) + + final_resp_tuple = None + while not self._state.finished and not self._state.invalid: + final_resp_tuple = await self._transmit_chunk( + sess, reader_fn, computed_size, progress_queue, stream_obj + ) + + if final_resp_tuple is None: + raise ValueError( + "Upload completed without receiving a final response." + ) + + _, _, body_bytes = final_resp_tuple + self._response = self._format_response(body_bytes) + progress_queue.put_nowait(_DONE_SENTINEL) + return self._response + except Exception as exc: + if hasattr(exc, "__dict__"): + exc.upload_url = self.upload_url + exc.chunk_size = self.chunk_size + progress_queue.put_nowait(exc) + raise + + task = asyncio.create_task(_run()) + return AsyncUploadOperation( + task=task, session=self, progress_queue=progress_queue + ) + + def resume( + self, + upload_url: str, + stream: Union[AsyncIterable[bytes], BinaryIO, bytes, Iterable[bytes]], + size: Optional[int] = None, + chunk_size: Optional[int] = None, + transport: Optional[Any] = None, + ) -> AsyncUploadOperation: + """Resumes an existing upload asynchronously, returning an AsyncUploadOperation. + + Args: + upload_url: Established upload session URL. + stream: Data payload to resume uploading. + size: Total stream size in bytes, if known. + chunk_size: Optional chunk size override in bytes. + transport: Optional aiohttp client session. + + Returns: + An AsyncUploadOperation handle representing the resumed transfer. + + Raises: + ValueError: If transport is missing. + """ + self._ensure_aiohttp() + sess = transport or self._transport + if sess is None: + raise ValueError("An aiohttp.ClientSession transport must be provided.") + + if chunk_size is not None: + self._state._chunk_size = chunk_size + + self._state._resumable_url = upload_url + progress_queue: asyncio.Queue = asyncio.Queue() + + async def _run(): + try: + reader_fn, computed_size, stream_obj = self._prepare_async_reader( + stream, size + ) + await self._recover(sess, stream_obj) + self._notify_progress( + common.ProgressState.OFFSET_RECEIVED, progress_queue + ) + + final_resp_tuple = None + while not self._state.finished and not self._state.invalid: + final_resp_tuple = await self._transmit_chunk( + sess, reader_fn, computed_size, progress_queue, stream_obj + ) + + if final_resp_tuple is None: + raise ValueError( + "Upload resumed but completed without receiving a final response." + ) + + _, _, body_bytes = final_resp_tuple + self._response = self._format_response(body_bytes) + progress_queue.put_nowait(_DONE_SENTINEL) + return self._response + except Exception as exc: + if hasattr(exc, "__dict__"): + exc.upload_url = self.upload_url + exc.chunk_size = self.chunk_size + progress_queue.put_nowait(exc) + raise + + task = asyncio.create_task(_run()) + return AsyncUploadOperation( + task=task, session=self, progress_queue=progress_queue + ) + + def _prepare_async_reader( + self, + stream: Union[AsyncIterable[bytes], BinaryIO, bytes, Iterable[bytes]], + size: Optional[int], + ) -> Tuple[Callable[[int], Awaitable[bytes]], Optional[int], Any]: + """Creates an asynchronous byte reader and determines stream length. + + Args: + stream: Input payload (async iterable, binary stream, bytes, or iterable). + size: Explicit total size in bytes, if known. + + Returns: + Tuple of (async reader function, computed total size, underlying stream object). + + Raises: + TypeError: If the stream type is not supported. + """ + computed_size = size + stream_obj = None + + if isinstance(stream, bytes): + stream_obj = io.BytesIO(stream) + if computed_size is None: + computed_size = len(stream) + + async def reader(n: int) -> bytes: + return stream_obj.read(n) + + return reader, computed_size, stream_obj + + if hasattr(stream, "read") and inspect.iscoroutinefunction(stream.read): + # Native async reader (e.g. asyncio.StreamReader) + async def reader(n: int) -> bytes: + return await stream.read(n) # type: ignore + + return reader, computed_size, stream + + if hasattr(stream, "read"): + # Synchronous binary stream: offload blocking reads to worker thread + stream_obj = stream + if computed_size is None and hasattr(stream, "getbuffer"): + computed_size = stream.getbuffer().nbytes + + if hasattr(stream_obj, "tell"): + try: + self._start_stream_offset = stream_obj.tell() + except (OSError, AttributeError): + self._start_stream_offset = 0 + + async def reader(n: int) -> bytes: + return await asyncio.to_thread(stream.read, n) # type: ignore + + return reader, computed_size, stream_obj + + if hasattr(stream, "__aiter__"): + # Native AsyncIterable[bytes] + iterator = stream.__aiter__() + buffer = bytearray() + + async def reader(n: int) -> bytes: + while len(buffer) < n: + try: + chunk = await iterator.__anext__() + buffer.extend(chunk) + except StopAsyncIteration: + break + result = bytes(buffer[:n]) + del buffer[:n] + return result + + return reader, computed_size, None + + if isinstance(stream, Iterable): + # Synchronous Iterable[bytes]: offload to worker thread + iterator = iter(stream) + buffer = bytearray() + + def _next_chunk(): + try: + return next(iterator) + except StopIteration: + return None + + async def reader(n: int) -> bytes: + while len(buffer) < n: + chunk = await asyncio.to_thread(_next_chunk) + if chunk is None: + break + buffer.extend(chunk) + result = bytes(buffer[:n]) + del buffer[:n] + return result + + return reader, computed_size, None + + raise TypeError(f"Unsupported stream type: {type(stream)}") + + def _format_response(self, response_bytes: bytes) -> Any: + """Formats response bytes into protobuf message type if provided. + + Args: + response_bytes: Raw HTTP response body bytes from final chunk. + + Returns: + Deserialized protobuf message or the raw bytes response. + """ + return _format_response_payload(response_bytes, self._config.response_type) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py new file mode 100644 index 000000000000..55669b85f7b5 --- /dev/null +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py @@ -0,0 +1,310 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sans-I/O Resumable Upload protocol state machine.""" + +import logging +from typing import Dict, Mapping, Optional, Sequence, Tuple, Union + +from google.api_core import exceptions +from google.api_core.resumable_transfer import common + +_LOGGER = logging.getLogger(__name__) + + +class ProtocolState(object): + """Encapsulates the state and command formatting for Resumable Upload protocol.""" + + def __init__( + self, + upload_url: Optional[str] = None, + chunk_size: int = common.DEFAULT_CHUNK_SIZE, + resumable_url: Optional[str] = None, + ) -> None: + """Initializes the protocol state machine. + + Args: + upload_url: The initial endpoint URL for starting the upload. + chunk_size: Desired chunk size in bytes. + resumable_url: Established upload session URL if resuming. + """ + self._initial_url = upload_url or "" + self._chunk_size = chunk_size + self._resumable_url = resumable_url + self._chunk_granularity: Optional[int] = None + self._bytes_uploaded = 0 + self._total_bytes: Optional[int] = None + self._finished = False + self._invalid = False + + @property + def initial_url(self) -> str: + """The initial endpoint URL for starting the upload.""" + return self._initial_url + + @property + def resumable_url(self) -> Optional[str]: + """The established upload session URL, or None if not established.""" + return self._resumable_url + + @property + def bytes_uploaded(self) -> int: + """The confirmed number of bytes committed to the server.""" + return self._bytes_uploaded + + @property + def total_bytes(self) -> Optional[int]: + """The total payload size in bytes, or None if unknown.""" + return self._total_bytes + + @property + def finished(self) -> bool: + """Whether the upload has completed successfully.""" + return self._finished + + @property + def invalid(self) -> bool: + """Whether the upload session has encountered a terminal failure.""" + return self._invalid + + @property + def chunk_size(self) -> int: + """Block-aligned chunk size informed by server granularity.""" + if self._chunk_granularity: + return ( + (self._chunk_size + self._chunk_granularity - 1) + // self._chunk_granularity + ) * self._chunk_granularity + return self._chunk_size + + def build_start_request( + self, + body: Union[str, bytes] = "", + headers: Optional[Sequence[Tuple[str, str]]] = None, + content_type: Optional[str] = None, + size: Optional[int] = None, + ) -> Tuple[str, str, Dict[str, str], bytes]: + """Formats the HTTP start request. + + Args: + body: Initial metadata payload. + headers: Optional sequence of header tuples to include. + content_type: MIME type of the stream payload. + size: Total size of the stream in bytes, if known. + + Returns: + A tuple of (HTTP method, URL, headers dict, payload bytes). + """ + self._total_bytes = size + req_headers: Dict[str, str] = {} + + if headers: + for k, v in headers: + req_headers[k] = v.decode("utf-8") if isinstance(v, bytes) else str(v) + + req_headers[common.HEADER_PROTOCOL] = common.PROTOCOL_RESUMABLE + req_headers[common.HEADER_COMMAND] = common.Command.START.value + + if content_type is not None: + req_headers[common.HEADER_CONTENT_TYPE] = content_type + if size is not None: + req_headers[common.HEADER_CONTENT_LENGTH] = str(size) + + payload = body.encode("utf-8") if isinstance(body, str) else body + return "POST", self._initial_url, req_headers, payload + + def process_start_response( + self, status_code: int, headers: Mapping[str, str] + ) -> str: + """Processes start response and extracts upload session URL. + + Args: + status_code: HTTP response status code. + headers: HTTP response headers. + + Returns: + The established resumable upload session URL. + + Raises: + exceptions.MissingStatusHeaderError: If status header is missing. + ValueError: If start response indicates failure or URL is missing. + """ + if status_code not in (200, 201): + self._invalid = True + raise ValueError(f"Start command failed with status {status_code}") + + headers_lower = {k.lower(): v for k, v in headers.items()} + status = headers_lower.get(common.HEADER_STATUS.lower()) + if not status: + raise exceptions.MissingStatusHeaderError( + f"Missing {common.HEADER_STATUS} header in start response" + ) + + resumable_url = headers_lower.get(common.HEADER_URL.lower()) + if not resumable_url: + self._invalid = True + raise ValueError(f"Server did not return {common.HEADER_URL} header") + + self._resumable_url = resumable_url + granularity = headers_lower.get(common.HEADER_CHUNK_GRANULARITY.lower()) + if granularity: + self._chunk_granularity = int(granularity) + + return self._resumable_url + + def build_chunk_request( + self, + data: Union[bytes, memoryview], + is_last_chunk: bool, + content_type: Optional[str] = None, + ) -> Tuple[str, str, Dict[str, str], bytes]: + """Formats an upload chunk request. + + Args: + data: Chunk byte data or memoryview slice. + is_last_chunk: True if this chunk concludes the upload payload. + content_type: MIME type of the uploaded chunk data. + + Returns: + A tuple of (HTTP method, URL, headers dict, payload bytes). + + Raises: + ValueError: If upload session URL is not established. + """ + if not self._resumable_url: + raise ValueError("Upload session URL not established.") + + command = ( + f"{common.Command.UPLOAD.value}, {common.Command.FINALIZE.value}" + if is_last_chunk + else common.Command.UPLOAD.value + ) + + headers = { + common.HEADER_COMMAND: command, + common.HEADER_OFFSET: str(self._bytes_uploaded), + } + if content_type: + headers["Content-Type"] = content_type + + payload = bytes(data) if isinstance(data, memoryview) else data + return "POST", self._resumable_url, headers, payload + + def process_chunk_response( + self, status_code: int, headers: Mapping[str, str], chunk_bytes_sent: int + ) -> None: + """Processes upload chunk response and updates committed bytes. + + Args: + status_code: HTTP response status code. + headers: HTTP response headers. + chunk_bytes_sent: Byte length of the chunk sent in the request. + + Raises: + exceptions.MissingStatusHeaderError: If status header is missing from successful response. + exceptions.UploadCancelledError: If server indicates session was cancelled. + """ + if status_code not in (200, 201): + return + + headers_lower = {k.lower(): v for k, v in headers.items()} + status = headers_lower.get(common.HEADER_STATUS.lower()) + if not status: + raise exceptions.MissingStatusHeaderError( + f"Missing {common.HEADER_STATUS} header in chunk upload response" + ) + + if status == common.Status.ACTIVE.value: + self._bytes_uploaded += chunk_bytes_sent + elif status == common.Status.FINAL.value: + self._finished = True + self._bytes_uploaded += chunk_bytes_sent + elif status == common.Status.CANCELLED.value: + self._invalid = True + raise exceptions.UploadCancelledError("Upload session was cancelled by server") + + def build_query_request(self) -> Tuple[str, str, Dict[str, str], bytes]: + """Formats the query request to discover server offset during recovery. + + Returns: + A tuple of (HTTP method, URL, headers dict, payload bytes). + + Raises: + ValueError: If upload session URL is not established. + """ + if not self._resumable_url: + raise ValueError("Upload session URL not established.") + + headers = {common.HEADER_COMMAND: common.Command.QUERY.value} + return "POST", self._resumable_url, headers, b"" + + def process_query_response( + self, status_code: int, headers: Mapping[str, str] + ) -> int: + """Processes query response and returns current server byte offset. + + Args: + status_code: HTTP response status code. + headers: HTTP response headers. + + Returns: + The current server byte offset. + + Raises: + ValueError: If query recovery indicates failure. + exceptions.UploadCancelledError: If server indicates session was cancelled. + """ + if status_code not in (200, 201): + self._invalid = True + raise ValueError(f"Query recovery failed with status {status_code}") + + headers_lower = {k.lower(): v for k, v in headers.items()} + status = headers_lower.get(common.HEADER_STATUS.lower()) + + if status == common.Status.ACTIVE.value: + received = int(headers_lower.get(common.HEADER_SIZE_RECEIVED.lower(), "0")) + self._bytes_uploaded = received + elif status == common.Status.FINAL.value: + self._finished = True + elif status == common.Status.CANCELLED.value: + self._invalid = True + raise exceptions.UploadCancelledError("Upload session was cancelled by server") + + return self._bytes_uploaded + + def build_cancel_request(self) -> Tuple[str, str, Dict[str, str], bytes]: + """Formats the cancel request. + + Returns: + A tuple of (HTTP method, URL, headers dict, payload bytes). + + Raises: + ValueError: If upload session URL is not established. + """ + if not self._resumable_url: + raise ValueError("Upload session URL not established.") + + headers = {common.HEADER_COMMAND: common.Command.CANCEL.value} + return "POST", self._resumable_url, headers, b"" + + def process_cancel_response( + self, status_code: int, headers: Mapping[str, str] + ) -> None: + """Processes cancel response and marks session invalid. + + Args: + status_code: HTTP response status code. + headers: HTTP response headers. + """ + self._invalid = True diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py new file mode 100644 index 000000000000..31370b6b8c1d --- /dev/null +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -0,0 +1,992 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Asynchronous tests for Resumable Upload protocol implementation.""" + +import asyncio +import datetime +import io +import json +from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Tuple, Union +from unittest import mock + +import pytest + +try: + import aiohttp # noqa: F401 + import google.auth.aio.transport # noqa: F401 + + GOOGLE_AUTH_AIO_INSTALLED = True +except ImportError: + GOOGLE_AUTH_AIO_INSTALLED = False + + +@pytest.fixture(autouse=True) +def check_async_rest_installed(request: pytest.FixtureRequest) -> None: + if request.node.name == "test_async_ensure_aiohttp_missing": + return + if not GOOGLE_AUTH_AIO_INSTALLED: + pytest.skip("Skipped because google-api-core[async_rest] is not installed") + + +from google.protobuf import empty_pb2 +import proto + +from tests.helpers import EchoResponse +from google.api_core import exceptions +from google.api_core.resumable_transfer import ( + AsyncResumableUploadSession, + AsyncUploadOperation, + MissingStatusHeaderError, + ProgressState, + ResumableUploadConfig, + TransferStalledError, + UnseekableStreamError, + UploadCancelledError, + UploadProgress, + common, + upload_async, +) + + +class DummyResponse: + """Mock response class representing a deserialized protobuf message.""" + + def __init__(self, name: str, size: int) -> None: + """Initializes a DummyResponse. + + Args: + name: Resource name string. + size: Resource size in bytes. + """ + self.name = name + self.size = size + + @classmethod + def from_json(cls, data: Union[str, bytes]) -> "DummyResponse": + """Deserializes JSON payload into a DummyResponse instance. + + Args: + data: JSON byte string or text. + + Returns: + A DummyResponse instance. + """ + d = json.loads(data.decode("utf-8") if isinstance(data, bytes) else data) + return cls(name=d.get("name", ""), size=d.get("size", 0)) + + +class DummyAsyncResponse: + """Mock HTTP response conforming to aiohttp.ClientResponse interface.""" + + def __init__( + self, + status: int = 200, + headers: Optional[Mapping[str, str]] = None, + body: bytes = b"", + ) -> None: + """Initializes a DummyAsyncResponse. + + Args: + status: HTTP status code. + headers: HTTP response headers mapping. + body: Response payload bytes. + """ + self.status = status + self.headers = headers or {} + self._body = body + + async def read(self) -> bytes: + """Reads and returns response payload bytes. + + Returns: + Raw response payload bytes. + """ + return self._body + + async def __aenter__(self) -> "DummyAsyncResponse": + """Enters the asynchronous context manager. + + Returns: + The DummyAsyncResponse instance. + """ + return self + + async def __aexit__( + self, exc_type: Any, exc_val: Any, exc_tb: Any + ) -> None: + """Exits the asynchronous context manager. + + Args: + exc_type: Exception type if raised. + exc_val: Exception value if raised. + exc_tb: Exception traceback if raised. + """ + pass + + +class DummyAsyncSession: + """Mock asynchronous HTTP client session conforming to aiohttp.ClientSession interface.""" + + def __init__( + self, responses: Optional[List[DummyAsyncResponse]] = None + ) -> None: + """Initializes a DummyAsyncSession. + + Args: + responses: Sequence of canned DummyAsyncResponse objects. + """ + self._responses: List[DummyAsyncResponse] = list(responses or []) + self.requests: List[Tuple[str, str, Dict[str, Any]]] = [] + + def request( + self, method: str, url: str, **kwargs: Any + ) -> DummyAsyncResponse: + """Records the request and yields the next canned response. + + Args: + method: HTTP method verb. + url: Destination endpoint URL. + **kwargs: Additional request parameters. + + Returns: + Next canned DummyAsyncResponse. + """ + self.requests.append((method, url, kwargs)) + if not self._responses: + return DummyAsyncResponse(status=200, headers={}, body=b"") + return self._responses.pop(0) + + +class NonSeekableBytesIO(io.BytesIO): + """BytesIO stream simulation with seekable returning False.""" + + def seekable(self) -> bool: + """Reports whether the stream supports random access. + + Returns: + False unconditionally. + """ + return False + + +# ===================================================================== +# 1. Initialization and Configuration Tests +# ===================================================================== + + +def test_async_session_initialization_defaults() -> None: + """Validates default attribute values of an uninitiated async session.""" + session = AsyncResumableUploadSession(upload_url="https://api.example.com/start") + assert session.upload_url is None + assert session.chunk_size == common.DEFAULT_CHUNK_SIZE + assert session.response is None + assert session.bytes_uploaded == 0 + assert session.finished is False + + +def test_async_missing_transport_raises() -> None: + """Verifies that invoking session operations without a transport raises ValueError.""" + session = AsyncResumableUploadSession() + + with pytest.raises(ValueError, match="aiohttp.ClientSession"): + session.upload(stream=b"data") + + with pytest.raises(ValueError, match="aiohttp.ClientSession"): + session.resume(upload_url="https://upload.example.com", stream=b"data") + + +@pytest.mark.asyncio +async def test_async_cancel_missing_transport_raises() -> None: + """Verifies that cancel without a transport raises ValueError.""" + session = AsyncResumableUploadSession() + with pytest.raises(ValueError, match="aiohttp.ClientSession"): + await session.cancel() + + +def test_async_ensure_aiohttp_missing(monkeypatch: pytest.MonkeyPatch) -> None: + """Verifies that _ensure_aiohttp raises ImportError when aiohttp is unavailable.""" + monkeypatch.setattr(upload_async, "aiohttp", None) + session = AsyncResumableUploadSession() + with pytest.raises(ImportError, match="google-api-core\\[async_rest\\]"): + session._ensure_aiohttp() + + +# ===================================================================== +# 2. Upload Execution and Operation Handle Tests +# ===================================================================== + + +@pytest.mark.asyncio +async def test_async_upload_direct_execution() -> None: + """Verifies single-chunk upload execution with protobuf deserialization.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "async_file.txt", "size": 10}', + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + config = ResumableUploadConfig(response_type=DummyResponse) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=async_transport, + ) + + result = await session.upload( + stream=b"0123456789", request_body='{"name": "test"}' + ) + + assert isinstance(result, DummyResponse) + assert result.name == "async_file.txt" + assert result.size == 10 + assert session.finished is True + assert session.bytes_uploaded == 10 + assert session.upload_url == "https://upload.example.com/resumable-async" + + +@pytest.mark.asyncio +async def test_async_upload_multi_chunk_operation_handle() -> None: + """Verifies multi-chunk upload dispatching and AsyncUploadOperation property handles.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk1_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "active"}, + body=b"", + ) + chunk2_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "multi.txt", "size": 8}', + ) + + async_transport = DummyAsyncSession([start_resp, chunk1_resp, chunk2_resp]) + config = ResumableUploadConfig(chunk_size=4, response_type=DummyResponse) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=async_transport, + ) + + upload_op = session.upload(stream=b"12345678") + assert isinstance(upload_op, AsyncUploadOperation) + assert upload_op.chunk_size == 4 + + result = await upload_op + assert isinstance(result, DummyResponse) + assert result.name == "multi.txt" + assert upload_op.response == result + assert upload_op.bytes_uploaded == 8 + assert upload_op.upload_url == "https://upload.example.com/resumable-async" + + # Validate commands dispatched in request history + assert len(async_transport.requests) == 3 + # Start request + assert async_transport.requests[0][2]["headers"]["X-Goog-Upload-Command"] == "start" + # Chunk 1 request + assert async_transport.requests[1][2]["headers"]["X-Goog-Upload-Command"] == "upload" + assert async_transport.requests[1][2]["headers"]["X-Goog-Upload-Offset"] == "0" + # Chunk 2 request (last chunk concludes transfer) + assert async_transport.requests[2][2]["headers"]["X-Goog-Upload-Command"] == "upload, finalize" + assert async_transport.requests[2][2]["headers"]["X-Goog-Upload-Offset"] == "4" + + +@pytest.mark.asyncio +async def test_async_upload_progress_tracking() -> None: + """Verifies that progress stream yields snapshots matching transmission milestones.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk1_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "active"}, + body=b"", + ) + chunk2_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "progress.txt", "size": 8}', + ) + + async_transport = DummyAsyncSession([start_resp, chunk1_resp, chunk2_resp]) + config = ResumableUploadConfig(chunk_size=4, response_type=DummyResponse) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=async_transport, + ) + + upload_op = session.upload(stream=b"12345678") + progress_list: List[UploadProgress] = [] + async for p in upload_op.progress(): + progress_list.append(p) + + final_resp = await upload_op + assert isinstance(final_resp, DummyResponse) + assert len(progress_list) == 3 + assert progress_list[0].state == ProgressState.STARTED + assert progress_list[1].state == ProgressState.UPLOADING + assert progress_list[1].bytes_uploaded == 4 + assert progress_list[2].state == ProgressState.FINALIZED + assert progress_list[2].bytes_uploaded == 8 + + +# ===================================================================== +# 3. Stream Input Types Tests +# ===================================================================== + + +@pytest.mark.asyncio +async def test_async_stream_types_async_iterable() -> None: + """Verifies upload compatibility with an asynchronous generator stream.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "async_gen.txt", "size": 6}', + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + + async def async_generator() -> AsyncIterator[bytes]: + yield b"abc" + yield b"def" + + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=DummyResponse), + transport=async_transport, + ) + + resp = await session.upload(stream=async_generator()) + assert resp.name == "async_gen.txt" + assert session.bytes_uploaded == 6 + + +@pytest.mark.asyncio +async def test_async_stream_types_binary_io() -> None: + """Verifies upload compatibility with a seekable BinaryIO stream.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "bytes_io.txt", "size": 5}', + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=DummyResponse), + transport=async_transport, + ) + + stream = io.BytesIO(b"hello") + resp = await session.upload(stream=stream) + assert resp.name == "bytes_io.txt" + assert session.bytes_uploaded == 5 + + +@pytest.mark.asyncio +async def test_async_stream_types_sync_iterable() -> None: + """Verifies upload compatibility with a synchronous iterable of byte chunks.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "iterable.txt", "size": 6}', + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=DummyResponse), + transport=async_transport, + ) + + resp = await session.upload(stream=[b"foo", b"bar"]) + assert resp.name == "iterable.txt" + assert session.bytes_uploaded == 6 + + +def test_async_stream_types_unsupported_raises() -> None: + """Verifies that passing an unsupported stream type raises TypeError.""" + session = AsyncResumableUploadSession(transport=DummyAsyncSession()) + with pytest.raises(TypeError, match="Unsupported stream type"): + session._prepare_async_reader(stream=12345, size=10) # type: ignore + + +# ===================================================================== +# 4. Resume and Offset Recovery Tests +# ===================================================================== + + +@pytest.mark.asyncio +async def test_async_resume_success() -> None: + """Verifies resuming an existing upload by querying server offset.""" + query_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "5", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "resumed_async.txt", "size": 10}', + ) + + async_transport = DummyAsyncSession([query_resp, chunk_resp]) + session = AsyncResumableUploadSession( + config=ResumableUploadConfig(response_type=DummyResponse), + transport=async_transport, + ) + + upload_op = session.resume( + upload_url="https://upload.example.com/resumable-async", + stream=b"0123456789", + ) + resp = await upload_op + assert resp.name == "resumed_async.txt" + assert session.bytes_uploaded == 10 + assert session.finished is True + + # First request was query + assert async_transport.requests[0][2]["headers"]["X-Goog-Upload-Command"] == "query" + # Second request was remaining chunk starting from offset 5 + assert async_transport.requests[1][2]["headers"]["X-Goog-Upload-Offset"] == "5" + + +@pytest.mark.asyncio +async def test_async_resume_recovery_unseekable_stream_raises() -> None: + """Verifies that UnseekableStreamError is raised if server offset cannot be rewound.""" + query_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "100", + }, + body=b"", + ) + + async_transport = DummyAsyncSession([query_resp]) + session = AsyncResumableUploadSession( + config=ResumableUploadConfig(response_type=DummyResponse), + transport=async_transport, + ) + + stream = NonSeekableBytesIO(b"some content") + upload_op = session.resume( + upload_url="https://upload.example.com/resumable-async", + stream=stream, + ) + + with pytest.raises(UnseekableStreamError) as exc_info: + await upload_op + + assert exc_info.value.upload_url == "https://upload.example.com/resumable-async" + + +@pytest.mark.asyncio +async def test_async_resume_recovery_seekable_stream() -> None: + """Verifies that seekable streams are rewound to committed server offset.""" + query_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "3", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "seekable.txt", "size": 6}', + ) + + async_transport = DummyAsyncSession([query_resp, chunk_resp]) + session = AsyncResumableUploadSession( + config=ResumableUploadConfig(response_type=DummyResponse), + transport=async_transport, + ) + + stream = io.BytesIO(b"abcdef") + upload_op = session.resume( + upload_url="https://upload.example.com/resumable-async", + stream=stream, + ) + resp = await upload_op + assert resp.name == "seekable.txt" + assert session.bytes_uploaded == 6 + + +# ===================================================================== +# 5. Cancellation Tests +# ===================================================================== + + +@pytest.mark.asyncio +async def test_async_cancel_success() -> None: + """Verifies client-initiated cancellation marks session state invalid.""" + cancel_resp = DummyAsyncResponse(status=200, headers={}, body=b"") + async_transport = DummyAsyncSession([cancel_resp]) + + session = AsyncResumableUploadSession( + resumable_url="https://upload.example.com/resumable-async", + transport=async_transport, + ) + await session.cancel() + assert session._state.invalid is True + assert async_transport.requests[0][2]["headers"]["X-Goog-Upload-Command"] == "cancel" + + +@pytest.mark.asyncio +async def test_async_server_cancelled_raises_error() -> None: + """Verifies that server returning cancelled status raises UploadCancelledError.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "cancelled"}, + body=b"", + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + ) + + with pytest.raises(UploadCancelledError, match="cancelled by server"): + await session.upload(stream=b"12345") + + +# ===================================================================== +# 6. Retry and Error Handling Tests +# ===================================================================== + + +@pytest.mark.asyncio +async def test_async_retry_transient_http_errors() -> None: + """Verifies transparent retries on transient HTTP status codes (503 Service Unavailable).""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_503 = DummyAsyncResponse(status=503, headers={}, body=b"Service Unavailable") + chunk_success = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "retried.txt", "size": 5}', + ) + + async_transport = DummyAsyncSession([start_resp, chunk_503, chunk_success]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=DummyResponse), + transport=async_transport, + ) + + with mock.patch("asyncio.sleep", new_callable=mock.AsyncMock): + resp = await session.upload(stream=b"hello") + + assert resp.name == "retried.txt" + assert session.finished is True + + +@pytest.mark.asyncio +async def test_async_non_retryable_error_raises() -> None: + """Verifies that non-retryable errors (e.g. 404 Not Found) terminate immediately.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_404 = DummyAsyncResponse(status=404, headers={}, body=b"Not Found") + + async_transport = DummyAsyncSession([start_resp, chunk_404]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + ) + + with pytest.raises(exceptions.NotFound): + await session.upload(stream=b"test") + + +@pytest.mark.asyncio +async def test_async_recoverable_status_code_triggers_recovery() -> None: + """Verifies that recoverable error status codes trigger query and offset reconciliation.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + # Chunk 1 returns 409 Conflict + chunk_conflict = DummyAsyncResponse(status=409, headers={}, body=b"Conflict") + # Recovery query returns confirmed committed offset 0 + query_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "0", + }, + body=b"", + ) + # Retransmission succeeds + chunk_success = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "recovered.txt", "size": 5}', + ) + + async_transport = DummyAsyncSession( + [start_resp, chunk_conflict, query_resp, chunk_success] + ) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=DummyResponse), + transport=async_transport, + ) + + with mock.patch("asyncio.sleep", new_callable=mock.AsyncMock): + resp = await session.upload(stream=b"hello") + + assert resp.name == "recovered.txt" + assert session.bytes_uploaded == 5 + + +@pytest.mark.asyncio +async def test_async_missing_status_header_triggers_recovery() -> None: + """Verifies that missing status header triggers query recovery.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + # Successful HTTP status but missing X-Goog-Upload-Status header + chunk_missing_hdr = DummyAsyncResponse(status=200, headers={}, body=b"") + # Recovery query + query_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "0", + }, + body=b"", + ) + chunk_success = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "header_recovered.txt", "size": 5}', + ) + + async_transport = DummyAsyncSession( + [start_resp, chunk_missing_hdr, query_resp, chunk_success] + ) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=DummyResponse), + transport=async_transport, + ) + + with mock.patch("asyncio.sleep", new_callable=mock.AsyncMock): + resp = await session.upload(stream=b"hello") + + assert resp.name == "header_recovered.txt" + + +# ===================================================================== +# 7. Stall Control and Deadline Tests +# ===================================================================== + + +@pytest.mark.asyncio +async def test_async_stall_timeout_raises_transfer_stalled_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Verifies that transfer stalling beyond timeout threshold raises TransferStalledError.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"name": "slow.txt", "size": 10}', + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + # Expect 100 bytes/sec, timeout 1 second + config = ResumableUploadConfig(stall_minimum_rate=100, stall_timeout=1.0) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=async_transport, + ) + + # Simulate elapsed time 10.0 seconds during 10-byte upload (rate = 1 byte/s < 100) + clock_vals = iter([0.0, 10.0, 10.0, 10.0]) + monkeypatch.setattr(upload_async, "_monotonic_clock", lambda: next(clock_vals)) + + with pytest.raises(TransferStalledError, match="Upload stalled"): + await session.upload(stream=b"0123456789") + + +@pytest.mark.asyncio +async def test_async_deadline_exceeded() -> None: + """Verifies that exceeding the configured upload deadline raises DeadlineExceeded.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + async_transport = DummyAsyncSession([start_resp]) + past_deadline = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + seconds=10 + ) + config = ResumableUploadConfig(deadline=past_deadline) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=async_transport, + ) + + with pytest.raises(exceptions.DeadlineExceeded): + await session.upload(stream=b"data") + + +# ===================================================================== +# 8. Response Deserialization Types Tests +# ===================================================================== + + +@pytest.mark.asyncio +async def test_async_response_type_proto_message() -> None: + """Verifies that a proto.Message type parses final response bytes.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b'{"content": "proto_async_payload"}', + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=EchoResponse), + transport=async_transport, + ) + + result = await session.upload(stream=b"data") + assert isinstance(result, EchoResponse) + assert result.content == "proto_async_payload" + + +@pytest.mark.asyncio +async def test_async_response_type_protobuf_message() -> None: + """Verifies that a google.protobuf.message.Message type parses final response bytes.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b"{}", + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=empty_pb2.Empty), + transport=async_transport, + ) + + result = await session.upload(stream=b"data") + assert isinstance(result, empty_pb2.Empty) + + +@pytest.mark.asyncio +async def test_async_response_type_callable() -> None: + """Verifies that a custom callable deserializer parses final response body bytes.""" + if not GOOGLE_AUTH_AIO_INSTALLED: + pytest.skip("Skipped because google-api-core[async_rest] is not installed") + + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b"parsed:hello", + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + + def custom_parser(raw: bytes) -> str: + return raw.decode("utf-8").upper() + + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=custom_parser), + transport=async_transport, + ) + + result = await session.upload(stream=b"data") + assert result == "PARSED:HELLO" + + +@pytest.mark.asyncio +async def test_async_response_type_raw_bytes() -> None: + """Verifies that raw bytes are returned when response_type is not configured.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b"raw-bytes-output", + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + ) + + result = await session.upload(stream=b"data") + assert result == b"raw-bytes-output" + + +@pytest.mark.asyncio +async def test_async_operation_error_propagation_in_progress() -> None: + """Verifies that background task errors propagate through progress queue iteration.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=403, + headers={}, + body=b"Permission Denied", + ) + + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + ) + + upload_op = session.upload(stream=b"data") + with pytest.raises(exceptions.Forbidden): + async for _ in upload_op.progress(): + pass + + with pytest.raises(exceptions.Forbidden): + await upload_op diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py new file mode 100644 index 000000000000..56176d395878 --- /dev/null +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -0,0 +1,730 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +from typing import Union +from unittest import mock + +import pytest +import requests +from google.protobuf import empty_pb2 +import proto + +from tests.helpers import EchoResponse +from google.api_core import exceptions +from google.api_core.resumable_transfer import ( + DEFAULT_CHUNK_SIZE, + Command, + MissingStatusHeaderError, + ProgressState, + ResumableUploadConfig, + ResumableUploadSession, + Status, + TransferStalledError, + UnseekableStreamError, + UploadCancelledError, + UploadProgress, + common, + upload_state, +) + + +class DummyResponse: + """Mock response class representing a deserialized protobuf message.""" + + def __init__(self, name: str, size: int) -> None: + """Initializes a DummyResponse. + + Args: + name: Resource name string. + size: Resource size in bytes. + """ + self.name = name + self.size = size + + @classmethod + def from_json(cls, data: Union[str, bytes]) -> "DummyResponse": + """Deserializes JSON payload into a DummyResponse instance. + + Args: + data: JSON byte string or text. + + Returns: + A DummyResponse instance. + """ + import json + + d = json.loads(data.decode("utf-8") if isinstance(data, bytes) else data) + return cls(name=d.get("name", ""), size=d.get("size", 0)) + + +# ===================================================================== +# 1. Common Constants and Error Types +# ===================================================================== + + +def test_common_constants(): + assert DEFAULT_CHUNK_SIZE == 10 * 1024 * 1024 + assert common.HEADER_PROTOCOL == "X-Goog-Upload-Protocol" + assert common.HEADER_COMMAND == "X-Goog-Upload-Command" + assert common.HEADER_STATUS == "X-Goog-Upload-Status" + assert common.HEADER_URL == "X-Goog-Upload-URL" + assert common.HEADER_OFFSET == "X-Goog-Upload-Offset" + assert common.HEADER_SIZE_RECEIVED == "X-Goog-Upload-Size-Received" + assert common.PROTOCOL_RESUMABLE == "resumable" + + assert Command.START == "start" + assert Command.UPLOAD == "upload" + assert Command.FINALIZE == "finalize" + assert Command.QUERY == "query" + assert Command.CANCEL == "cancel" + + assert Status.ACTIVE == "active" + assert Status.FINAL == "final" + assert Status.CANCELLED == "cancelled" + + assert ProgressState.STARTED == "started" + assert ProgressState.UPLOADING == "uploading" + assert ProgressState.RECOVERING == "recovering" + assert ProgressState.OFFSET_RECEIVED == "offset received" + assert ProgressState.FINALIZED == "finalized" + + +def test_upload_progress_dataclass(): + prog = UploadProgress( + upload_url="https://upload.example.com/session123", + chunk_size=1024, + bytes_uploaded=512, + total_bytes=2048, + state=ProgressState.UPLOADING, + ) + assert prog.upload_url == "https://upload.example.com/session123" + assert prog.chunk_size == 1024 + assert prog.bytes_uploaded == 512 + assert prog.total_bytes == 2048 + assert prog.state == ProgressState.UPLOADING + + +def test_exception_hierarchy(): + assert issubclass(TransferStalledError, exceptions.GoogleAPICallError) + assert issubclass(UnseekableStreamError, exceptions.GoogleAPICallError) + assert issubclass(UploadCancelledError, exceptions.GoogleAPICallError) + assert issubclass(MissingStatusHeaderError, exceptions.GoogleAPICallError) + assert exceptions.TransferStalledError is TransferStalledError + assert exceptions.UnseekableStreamError is UnseekableStreamError + assert exceptions.UploadCancelledError is UploadCancelledError + assert exceptions.MissingStatusHeaderError is MissingStatusHeaderError + + +# ===================================================================== +# 2. Pure Sans-I/O State Machine (upload_state.py) +# ===================================================================== + + +def test_protocol_state_start_request(): + state = upload_state.ProtocolState(upload_url="https://api.example.com/start") + method, url, headers, payload = state.build_start_request( + body='{"name": "test"}', + headers=[("X-Custom", "val")], + content_type="text/plain", + size=1000, + ) + + assert method == "POST" + assert url == "https://api.example.com/start" + assert headers["X-Goog-Upload-Protocol"] == "resumable" + assert headers["X-Goog-Upload-Command"] == "start" + assert headers["X-Goog-Upload-Header-Content-Type"] == "text/plain" + assert headers["X-Goog-Upload-Header-Content-Length"] == "1000" + assert headers["X-Custom"] == "val" + assert payload == b'{"name": "test"}' + + +def test_protocol_state_process_start_response(): + state = upload_state.ProtocolState(upload_url="https://api.example.com/start") + headers = { + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-id", + "X-Goog-Upload-Chunk-Granularity": "262144", + } + url = state.process_start_response(200, headers) + assert url == "https://upload.example.com/resumable-id" + assert state.resumable_url == "https://upload.example.com/resumable-id" + assert state._chunk_granularity == 262144 + + +def test_protocol_state_start_response_missing_status(): + state = upload_state.ProtocolState(upload_url="https://api.example.com/start") + headers = {"X-Goog-Upload-URL": "https://upload.example.com/resumable-id"} + with pytest.raises(MissingStatusHeaderError): + state.process_start_response(200, headers) + + +def test_protocol_state_start_response_missing_url(): + state = upload_state.ProtocolState(upload_url="https://api.example.com/start") + headers = {"X-Goog-Upload-Status": "active"} + with pytest.raises(ValueError, match="Server did not return"): + state.process_start_response(200, headers) + + +def test_protocol_state_granularity_alignment(): + state = upload_state.ProtocolState(chunk_size=500) + assert state.chunk_size == 500 + state._chunk_granularity = 256 + # 500 rounded up to multiple of 256 is 512 + assert state.chunk_size == 512 + + +def test_protocol_state_chunk_request_and_response(): + state = upload_state.ProtocolState( + resumable_url="https://upload.example.com/session" + ) + + # First chunk: not last + method, url, headers, payload = state.build_chunk_request( + data=b"0123456789", is_last_chunk=False + ) + assert headers["X-Goog-Upload-Command"] == "upload" + assert headers["X-Goog-Upload-Offset"] == "0" + assert payload == b"0123456789" + + state.process_chunk_response(200, {"X-Goog-Upload-Status": "active"}, 10) + assert state.bytes_uploaded == 10 + assert not state.finished + + # Second chunk: last chunk + method, url, headers, payload = state.build_chunk_request( + data=b"abcdef", is_last_chunk=True, content_type="text/plain" + ) + assert headers["X-Goog-Upload-Command"] == "upload, finalize" + assert headers["X-Goog-Upload-Offset"] == "10" + assert headers["Content-Type"] == "text/plain" + + state.process_chunk_response(200, {"X-Goog-Upload-Status": "final"}, 6) + assert state.bytes_uploaded == 16 + assert state.finished + + +def test_protocol_state_chunk_missing_status_header(): + state = upload_state.ProtocolState( + resumable_url="https://upload.example.com/session" + ) + with pytest.raises(MissingStatusHeaderError): + state.process_chunk_response(200, {}, 10) + + +def test_protocol_state_query_and_cancel(): + state = upload_state.ProtocolState( + resumable_url="https://upload.example.com/session" + ) + method, url, headers, payload = state.build_query_request() + assert headers["X-Goog-Upload-Command"] == "query" + + received = state.process_query_response( + 200, {"X-Goog-Upload-Status": "active", "X-Goog-Upload-Size-Received": "1024"} + ) + assert received == 1024 + assert state.bytes_uploaded == 1024 + + method, url, headers, payload = state.build_cancel_request() + assert headers["X-Goog-Upload-Command"] == "cancel" + state.process_cancel_response(200, {}) + assert state.invalid + + +# ===================================================================== +# 3. ResumableUploadConfig Sensible Defaults +# ===================================================================== + + +def test_resumable_upload_config_defaults(): + config = ResumableUploadConfig() + assert config.chunk_size == 10 * 1024 * 1024 + assert config.stall_minimum_rate == 64 * 1024 + assert config.stall_timeout == 120.0 + assert config.start_timeout is None + assert config.start_retry is None + assert config.additional_headers is None + assert config.deadline is None + + +def test_resumable_upload_config_fallbacks_and_headers(): + retry1 = mock.Mock() + config1 = ResumableUploadConfig( + start_timeout=45.0, + retry=retry1, + additional_headers={"X-Test": "1"}, + ) + assert config1.timeout == 45.0 + assert config1.start_timeout == 45.0 + assert config1.retry is retry1 + assert config1.start_retry is retry1 + assert config1.start_headers == [("X-Test", "1")] + + retry2 = mock.Mock() + config2 = ResumableUploadConfig( + timeout=30.0, + start_retry=retry2, + additional_headers=[("X-Test", "2")], + ) + assert config2.timeout == 30.0 + assert config2.start_timeout == 30.0 + assert config2.retry is retry2 + assert config2.start_retry is retry2 + assert config2.start_headers == [("X-Test", "2")] + + +# ===================================================================== +# 4. Synchronous ResumableUploadSession (upload.py) +# ===================================================================== + + +def test_sync_upload_direct_execution(): + session_transport = mock.create_autospec(requests.Session, instance=True) + + # 1. Start response + start_resp = mock.Mock() + start_resp.ok = True + start_resp.status_code = 200 + start_resp.headers = { + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + } + + # 2. Chunk response + chunk_resp = mock.Mock() + chunk_resp.ok = True + chunk_resp.status_code = 200 + chunk_resp.headers = {"X-Goog-Upload-Status": "final"} + chunk_resp.content = b'{"name": "done.txt", "size": 11}' + + session_transport.request.side_effect = [start_resp, chunk_resp] + + config = ResumableUploadConfig(response_type=DummyResponse) + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=session_transport, + ) + + payload = b"Hello world" + result = session.upload(stream=payload, request_body='{"name": "test"}') + + assert isinstance(result, DummyResponse) + assert result.name == "done.txt" + assert result.size == 11 + assert session.finished is True + assert session.bytes_uploaded == 11 + assert session.upload_url == "https://upload.example.com/resumable-123" + assert session.response == result + + +def test_sync_upload_iterative_progress(): + session_transport = mock.create_autospec(requests.Session, instance=True) + + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + chunk1_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "active"}, + ) + chunk2_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b'{"name": "stream.txt", "size": 8}', + ) + + session_transport.request.side_effect = [start_resp, chunk1_resp, chunk2_resp] + + config = ResumableUploadConfig(chunk_size=4, response_type=DummyResponse) + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=session_transport, + ) + + progress_events = list(session.iter_upload(stream=b"12345678")) + assert len(progress_events) == 3 + assert progress_events[0].state == ProgressState.STARTED + assert progress_events[1].state == ProgressState.UPLOADING + assert progress_events[1].bytes_uploaded == 4 + assert progress_events[2].state == ProgressState.FINALIZED + assert progress_events[2].bytes_uploaded == 8 + + assert session.response.name == "stream.txt" + assert session.response.size == 8 + + +def test_sync_resume(): + session_transport = mock.create_autospec(requests.Session, instance=True) + + # 1. Query response returns offset 5 + query_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "5", + }, + ) + # 2. Remaining chunk response + chunk_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b'{"name": "resumed.txt", "size": 10}', + ) + session_transport.request.side_effect = [query_resp, chunk_resp] + + config = ResumableUploadConfig(response_type=DummyResponse) + session = ResumableUploadSession(config=config) + + stream = io.BytesIO(b"0123456789") + resp = session.resume( + upload_url="https://upload.example.com/resumable-123", + stream=stream, + transport=session_transport, + ) + + assert isinstance(resp, DummyResponse) + assert resp.name == "resumed.txt" + assert session.bytes_uploaded == 10 + assert session.finished is True + + +def test_sync_iter_resume(): + """Verifies streaming progress during upload resumption.""" + session_transport = mock.create_autospec(requests.Session, instance=True) + + query_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "5", + }, + ) + chunk_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b'{"name": "iter_resumed.txt", "size": 10}', + ) + session_transport.request.side_effect = [query_resp, chunk_resp] + + config = ResumableUploadConfig(response_type=DummyResponse) + session = ResumableUploadSession(config=config) + + stream = io.BytesIO(b"0123456789") + progress_list = list( + session.iter_resume( + upload_url="https://upload.example.com/resumable-123", + stream=stream, + transport=session_transport, + ) + ) + + assert session.response.name == "iter_resumed.txt" + assert session.bytes_uploaded == 10 + assert session.finished is True + assert len(progress_list) == 2 + assert progress_list[0].state == ProgressState.OFFSET_RECEIVED + assert progress_list[1].state == ProgressState.FINALIZED + + +def test_sync_recoverable_status_code_triggers_offset_recovery(): + session_transport = mock.create_autospec(requests.Session, instance=True) + + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + # First chunk upload fails with 400 (recoverable Category 2) + err400_resp = mock.Mock( + ok=False, + status_code=400, + headers={}, + ) + err400_resp.json.return_value = {"error": {"message": "Bad Request", "details": []}} + err400_resp.text = '{"error": {"message": "Bad Request"}}' + err400_resp.content = err400_resp.text.encode("utf-8") + # Recovery query returns offset 0 + query_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "0", + }, + ) + # Retry chunk upload succeeds + success_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b'{"name": "recovered.txt", "size": 5}', + ) + + session_transport.request.side_effect = [ + start_resp, + err400_resp, + query_resp, + success_resp, + ] + + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=DummyResponse), + transport=session_transport, + ) + + resp = session.upload(stream=b"12345") + assert resp.name == "recovered.txt" + assert session.bytes_uploaded == 5 + + +def test_sync_exceptions_carry_upload_url_and_chunk_size(): + session_transport = mock.create_autospec(requests.Session, instance=True) + + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + # Fatal 403 error on chunk upload + err403_resp = mock.Mock( + ok=False, + status_code=403, + headers={}, + ) + err403_resp.json.return_value = {"error": {"message": "Forbidden", "details": []}} + err403_resp.text = '{"error": {"message": "Forbidden"}}' + err403_resp.content = err403_resp.text.encode("utf-8") + session_transport.request.side_effect = [start_resp, err403_resp] + + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + transport=session_transport, + ) + + with pytest.raises(exceptions.GoogleAPICallError) as exc_info: + session.upload(stream=b"data") + + err = exc_info.value + assert err.upload_url == "https://upload.example.com/resumable-123" + assert err.chunk_size == session.chunk_size + + +def test_sync_unseekable_stream_error_on_preceding_offset(): + session_transport = mock.create_autospec(requests.Session, instance=True) + + query_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "50", + }, + ) + session_transport.request.side_effect = [query_resp] + + session = ResumableUploadSession( + config=ResumableUploadConfig(), + transport=session_transport, + ) + + # Mock an unseekable stream + unseekable = mock.Mock(spec=io.RawIOBase) + unseekable.seekable.return_value = False + + with pytest.raises(UnseekableStreamError) as exc_info: + session.resume( + upload_url="https://upload.example.com/resumable-123", + stream=unseekable, + transport=session_transport, + ) + + assert exc_info.value.upload_url == "https://upload.example.com/resumable-123" + + +def test_sync_stall_control_timeout(): + session_transport = mock.create_autospec(requests.Session, instance=True) + + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + session_transport.request.side_effect = [ + start_resp, + requests.exceptions.Timeout("Read timed out"), + ] + + # Configure stall control with 0.1s timeout + config = ResumableUploadConfig( + stall_minimum_rate=1024, + stall_timeout=0.1, + ) + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=session_transport, + ) + + with pytest.raises(TransferStalledError) as exc_info: + session.upload(stream=b"test data") + + assert exc_info.value.upload_url == "https://upload.example.com/resumable-123" + + +def test_sync_cancel(): + session_transport = mock.create_autospec(requests.Session, instance=True) + cancel_resp = mock.Mock(ok=True, status_code=200, headers={}) + session_transport.request.return_value = cancel_resp + + session = ResumableUploadSession( + resumable_url="https://upload.example.com/resumable-123", + transport=session_transport, + ) + session.cancel() + assert session._state.invalid is True + + +def test_sync_response_type_proto_message(): + session_transport = mock.create_autospec(requests.Session, instance=True) + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + chunk_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b'{"content": "proto_payload"}', + ) + session_transport.request.side_effect = [start_resp, chunk_resp] + + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=EchoResponse), + transport=session_transport, + ) + resp = session.upload(stream=b"payload") + assert isinstance(resp, EchoResponse) + assert resp.content == "proto_payload" + + +def test_sync_response_type_protobuf_message(): + session_transport = mock.create_autospec(requests.Session, instance=True) + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + chunk_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b"{}", + ) + session_transport.request.side_effect = [start_resp, chunk_resp] + + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=empty_pb2.Empty), + transport=session_transport, + ) + resp = session.upload(stream=b"payload") + assert isinstance(resp, empty_pb2.Empty) + + +def test_sync_response_type_callable(): + session_transport = mock.create_autospec(requests.Session, instance=True) + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + chunk_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b"custom_payload", + ) + session_transport.request.side_effect = [start_resp, chunk_resp] + + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=lambda c: c.decode("utf-8").upper()), + transport=session_transport, + ) + resp = session.upload(stream=b"payload") + assert resp == "CUSTOM_PAYLOAD" + + +def test_sync_response_type_raw_response(): + session_transport = mock.create_autospec(requests.Session, instance=True) + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + chunk_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b"raw_content", + ) + session_transport.request.side_effect = [start_resp, chunk_resp] + + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(response_type=None), + transport=session_transport, + ) + resp = session.upload(stream=b"payload") + assert resp is chunk_resp + + From 326ae786d7a8fe0a39140977b019b9e1e86c7cec Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 11 Sep 2026 21:58:44 +0000 Subject: [PATCH 02/43] lint --- .../api_core/resumable_transfer/common.py | 1 - .../api_core/resumable_transfer/upload.py | 17 ++++------ .../resumable_transfer/upload_async.py | 1 + .../resumable_transfer/upload_state.py | 8 +++-- .../asyncio/test_resumable_transfer_async.py | 33 +++++++++---------- .../tests/unit/test_resumable_transfer.py | 6 ++-- 6 files changed, 32 insertions(+), 34 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/common.py b/packages/google-api-core/google/api_core/resumable_transfer/common.py index cc0046f0d6ca..0859cde83de3 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/common.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/common.py @@ -87,4 +87,3 @@ class UploadProgress: # HTTP status codes indicating state consistency errors requiring recovery RECOVERABLE_STATUS_CODES = (400, 409, 412, 416) - diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 57caaccafc71..811042ee3944 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -34,14 +34,14 @@ Union, ) +import google.protobuf.message +import proto import requests +from google.protobuf import json_format import google.api_core.retry -import google.protobuf.message -import proto from google.api_core import exceptions from google.api_core.resumable_transfer import common, upload_state -from google.protobuf import json_format _LOGGER = logging.getLogger(__name__) _DEFAULT_START_TIMEOUT = 60.0 # seconds for initial start request @@ -172,9 +172,7 @@ def finished(self) -> bool: """bool: Whether the upload has completed successfully.""" return self._state.finished - def _get_transport( - self, transport: Optional[requests.Session] - ) -> requests.Session: + def _get_transport(self, transport: Optional[requests.Session]) -> requests.Session: """Resolves the requests.Session transport. Args: @@ -283,6 +281,7 @@ def _get_retry_predicate(self) -> Callable[[Any], bool]: Returns: A callable accepting an exception and returning a boolean. """ + def should_retry(exc: Any) -> bool: if isinstance( exc, @@ -396,9 +395,7 @@ def _update_stall_control( else: self._stall_timeout_started = None - def _reposition_stream_offset( - self, stream: BinaryIO, received: int - ) -> int: + def _reposition_stream_offset(self, stream: BinaryIO, received: int) -> int: """Adjusts in-memory chunk buffer or seeks input stream to server offset. Args: @@ -490,6 +487,7 @@ def _transmit_chunk( Returns: The HTTP response for the transmitted chunk. """ + def do_transmit() -> requests.Response: chunk_size = self._state.chunk_size @@ -904,4 +902,3 @@ def _format_response_payload( return response_type(content) return response - diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 2a1c80dfade9..445b75257105 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -378,6 +378,7 @@ async def _transmit_chunk( DeadlineExceeded: If upload deadline is reached. GoogleAPICallError: If chunk upload encounters an unrecoverable error. """ + async def do_transmit(): chunk_size = self._state.chunk_size diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py index 55669b85f7b5..234e3e6c9526 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py @@ -232,7 +232,9 @@ def process_chunk_response( self._bytes_uploaded += chunk_bytes_sent elif status == common.Status.CANCELLED.value: self._invalid = True - raise exceptions.UploadCancelledError("Upload session was cancelled by server") + raise exceptions.UploadCancelledError( + "Upload session was cancelled by server" + ) def build_query_request(self) -> Tuple[str, str, Dict[str, str], bytes]: """Formats the query request to discover server offset during recovery. @@ -279,7 +281,9 @@ def process_query_response( self._finished = True elif status == common.Status.CANCELLED.value: self._invalid = True - raise exceptions.UploadCancelledError("Upload session was cancelled by server") + raise exceptions.UploadCancelledError( + "Upload session was cancelled by server" + ) return self._bytes_uploaded diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 31370b6b8c1d..859039d373a4 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -40,10 +40,9 @@ def check_async_rest_installed(request: pytest.FixtureRequest) -> None: pytest.skip("Skipped because google-api-core[async_rest] is not installed") -from google.protobuf import empty_pb2 import proto +from google.protobuf import empty_pb2 -from tests.helpers import EchoResponse from google.api_core import exceptions from google.api_core.resumable_transfer import ( AsyncResumableUploadSession, @@ -58,6 +57,7 @@ def check_async_rest_installed(request: pytest.FixtureRequest) -> None: common, upload_async, ) +from tests.helpers import EchoResponse class DummyResponse: @@ -123,9 +123,7 @@ async def __aenter__(self) -> "DummyAsyncResponse": """ return self - async def __aexit__( - self, exc_type: Any, exc_val: Any, exc_tb: Any - ) -> None: + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Exits the asynchronous context manager. Args: @@ -139,9 +137,7 @@ async def __aexit__( class DummyAsyncSession: """Mock asynchronous HTTP client session conforming to aiohttp.ClientSession interface.""" - def __init__( - self, responses: Optional[List[DummyAsyncResponse]] = None - ) -> None: + def __init__(self, responses: Optional[List[DummyAsyncResponse]] = None) -> None: """Initializes a DummyAsyncSession. Args: @@ -150,9 +146,7 @@ def __init__( self._responses: List[DummyAsyncResponse] = list(responses or []) self.requests: List[Tuple[str, str, Dict[str, Any]]] = [] - def request( - self, method: str, url: str, **kwargs: Any - ) -> DummyAsyncResponse: + def request(self, method: str, url: str, **kwargs: Any) -> DummyAsyncResponse: """Records the request and yields the next canned response. Args: @@ -253,9 +247,7 @@ async def test_async_upload_direct_execution() -> None: transport=async_transport, ) - result = await session.upload( - stream=b"0123456789", request_body='{"name": "test"}' - ) + result = await session.upload(stream=b"0123456789", request_body='{"name": "test"}') assert isinstance(result, DummyResponse) assert result.name == "async_file.txt" @@ -311,10 +303,15 @@ async def test_async_upload_multi_chunk_operation_handle() -> None: # Start request assert async_transport.requests[0][2]["headers"]["X-Goog-Upload-Command"] == "start" # Chunk 1 request - assert async_transport.requests[1][2]["headers"]["X-Goog-Upload-Command"] == "upload" + assert ( + async_transport.requests[1][2]["headers"]["X-Goog-Upload-Command"] == "upload" + ) assert async_transport.requests[1][2]["headers"]["X-Goog-Upload-Offset"] == "0" # Chunk 2 request (last chunk concludes transfer) - assert async_transport.requests[2][2]["headers"]["X-Goog-Upload-Command"] == "upload, finalize" + assert ( + async_transport.requests[2][2]["headers"]["X-Goog-Upload-Command"] + == "upload, finalize" + ) assert async_transport.requests[2][2]["headers"]["X-Goog-Upload-Offset"] == "4" @@ -591,7 +588,9 @@ async def test_async_cancel_success() -> None: ) await session.cancel() assert session._state.invalid is True - assert async_transport.requests[0][2]["headers"]["X-Goog-Upload-Command"] == "cancel" + assert ( + async_transport.requests[0][2]["headers"]["X-Goog-Upload-Command"] == "cancel" + ) @pytest.mark.asyncio diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 56176d395878..aa2a56c75f88 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -16,12 +16,11 @@ from typing import Union from unittest import mock +import proto import pytest import requests from google.protobuf import empty_pb2 -import proto -from tests.helpers import EchoResponse from google.api_core import exceptions from google.api_core.resumable_transfer import ( DEFAULT_CHUNK_SIZE, @@ -38,6 +37,7 @@ common, upload_state, ) +from tests.helpers import EchoResponse class DummyResponse: @@ -726,5 +726,3 @@ def test_sync_response_type_raw_response(): ) resp = session.upload(stream=b"payload") assert resp is chunk_resp - - From 9936e0d09c8571d9d67c56776bf343d276b1dd90 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 11 Sep 2026 22:07:39 +0000 Subject: [PATCH 03/43] lint --- .../asyncio/test_resumable_transfer_async.py | 37 ++++++++----------- .../tests/unit/test_resumable_transfer.py | 1 - 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 859039d373a4..28bb290939a3 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -14,7 +14,6 @@ """Asynchronous tests for Resumable Upload protocol implementation.""" -import asyncio import datetime import io import json @@ -22,32 +21,12 @@ from unittest import mock import pytest - -try: - import aiohttp # noqa: F401 - import google.auth.aio.transport # noqa: F401 - - GOOGLE_AUTH_AIO_INSTALLED = True -except ImportError: - GOOGLE_AUTH_AIO_INSTALLED = False - - -@pytest.fixture(autouse=True) -def check_async_rest_installed(request: pytest.FixtureRequest) -> None: - if request.node.name == "test_async_ensure_aiohttp_missing": - return - if not GOOGLE_AUTH_AIO_INSTALLED: - pytest.skip("Skipped because google-api-core[async_rest] is not installed") - - -import proto from google.protobuf import empty_pb2 from google.api_core import exceptions from google.api_core.resumable_transfer import ( AsyncResumableUploadSession, AsyncUploadOperation, - MissingStatusHeaderError, ProgressState, ResumableUploadConfig, TransferStalledError, @@ -59,6 +38,22 @@ def check_async_rest_installed(request: pytest.FixtureRequest) -> None: ) from tests.helpers import EchoResponse +try: + import aiohttp # noqa: F401 + import google.auth.aio.transport # noqa: F401 + + GOOGLE_AUTH_AIO_INSTALLED = True +except ImportError: + GOOGLE_AUTH_AIO_INSTALLED = False + + +@pytest.fixture(autouse=True) +def check_async_rest_installed(request: pytest.FixtureRequest) -> None: + if request.node.name == "test_async_ensure_aiohttp_missing": + return + if not GOOGLE_AUTH_AIO_INSTALLED: + pytest.skip("Skipped because google-api-core[async_rest] is not installed") + class DummyResponse: """Mock response class representing a deserialized protobuf message.""" diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index aa2a56c75f88..20c5532a8a84 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -16,7 +16,6 @@ from typing import Union from unittest import mock -import proto import pytest import requests from google.protobuf import empty_pb2 From 9e689b8bb414508b5c3d637d19a12fcd221ba6e3 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 15:34:04 +0000 Subject: [PATCH 04/43] mypy --- .../google/api_core/exceptions.py | 35 +++++--- .../api_core/resumable_transfer/__init__.py | 2 + .../api_core/resumable_transfer/upload.py | 41 +++++---- .../resumable_transfer/upload_async.py | 90 ++++++++++--------- 4 files changed, 93 insertions(+), 75 deletions(-) diff --git a/packages/google-api-core/google/api_core/exceptions.py b/packages/google-api-core/google/api_core/exceptions.py index d86c1c409f4a..aa9898c5b9a6 100644 --- a/packages/google-api-core/google/api_core/exceptions.py +++ b/packages/google-api-core/google/api_core/exceptions.py @@ -446,29 +446,40 @@ class AsyncRestUnsupportedParameterError(NotImplementedError): pass -class TransferStalledError(GoogleAPICallError): +class ResumableTransferError(GoogleAPICallError): + """Base class for resumable transfer errors.""" + + upload_url: Optional[str] = None + chunk_size: Optional[int] = None + + def __init__( + self, + message: str, + *args, + upload_url: Optional[str] = None, + chunk_size: Optional[int] = None, + **kwargs, + ) -> None: + super().__init__(message, *args, **kwargs) + self.upload_url = upload_url + self.chunk_size = chunk_size + + +class TransferStalledError(ResumableTransferError): """Raised when upload throughput stays below minimum rate past stall timeout.""" - pass - -class UnseekableStreamError(GoogleAPICallError): +class UnseekableStreamError(ResumableTransferError): """Raised when server recovery requires rewinding a non-seekable stream.""" - pass - -class UploadCancelledError(GoogleAPICallError): +class UploadCancelledError(ResumableTransferError): """Raised when the upload is cancelled by the client or server.""" - pass - -class MissingStatusHeaderError(GoogleAPICallError): +class MissingStatusHeaderError(ResumableTransferError): """Raised when server response lacks the required X-Goog-Upload-Status header.""" - pass - def exception_class_for_http_status(status_code): """Return the exception class for a specific HTTP status code. diff --git a/packages/google-api-core/google/api_core/resumable_transfer/__init__.py b/packages/google-api-core/google/api_core/resumable_transfer/__init__.py index 34217378f70a..041ef2a6cf72 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/__init__.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/__init__.py @@ -16,6 +16,7 @@ from google.api_core.exceptions import ( MissingStatusHeaderError, + ResumableTransferError, TransferStalledError, UnseekableStreamError, UploadCancelledError, @@ -41,6 +42,7 @@ "DEFAULT_CHUNK_SIZE", "MissingStatusHeaderError", "ProgressState", + "ResumableTransferError", "Status", "TransferStalledError", "UnseekableStreamError", diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 811042ee3944..e31bdff2b682 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -32,6 +32,7 @@ Sequence, Tuple, Union, + cast, ) import google.protobuf.message @@ -196,8 +197,8 @@ def _enrich_exception(self, exc: BaseException) -> None: exc: Exception instance to augment with upload_url and chunk_size. """ if hasattr(exc, "__dict__"): - exc.upload_url = self.upload_url - exc.chunk_size = self.chunk_size + setattr(exc, "upload_url", self.upload_url) + setattr(exc, "chunk_size", self.chunk_size) def _notify_progress(self, state: common.ProgressState) -> None: """Notifies registered progress callback with current upload status. @@ -390,7 +391,9 @@ def _update_stall_control( ) raise exceptions.TransferStalledError( f"Upload stalled: transfer rate remained below {rate} bytes/s " - f"for longer than {self._config.stall_timeout}s." + f"for longer than {self._config.stall_timeout}s.", + upload_url=self.upload_url, + chunk_size=self.chunk_size, ) else: self._stall_timeout_started = None @@ -419,19 +422,19 @@ def _reposition_stream_offset(self, stream: BinaryIO, received: int) -> int: self._buffered_chunk = None if hasattr(stream, "seekable") and not stream.seekable(): - err = exceptions.UnseekableStreamError( - f"Stream is not seekable. Cannot recover upload to offset {received}." + raise exceptions.UnseekableStreamError( + f"Stream is not seekable. Cannot recover upload to offset {received}.", + upload_url=self.upload_url, + chunk_size=self.chunk_size, ) - self._enrich_exception(err) - raise err try: stream.seek(self._start_stream_offset + received) except (OSError, AttributeError) as exc: - err = exceptions.UnseekableStreamError( - f"Failed to seek stream to offset {received}: {exc}" - ) - self._enrich_exception(err) - raise err from exc + raise exceptions.UnseekableStreamError( + f"Failed to seek stream to offset {received}: {exc}", + upload_url=self.upload_url, + chunk_size=self.chunk_size, + ) from exc return received @@ -551,11 +554,11 @@ def do_http() -> requests.Response: raise exceptions.DeadlineExceeded( f"Resumable upload deadline {self._config.deadline} exceeded." ) from exc - stalled_err = exceptions.TransferStalledError( - f"Upload stalled: chunk transfer timed out ({exc})." - ) - self._enrich_exception(stalled_err) - raise stalled_err from exc + raise exceptions.TransferStalledError( + f"Upload stalled: chunk transfer timed out ({exc}).", + upload_url=self.upload_url, + chunk_size=self.chunk_size, + ) from exc is_recoverable = ( isinstance(exc, exceptions.GoogleAPICallError) @@ -828,7 +831,7 @@ def _prepare_stream( if computed_size is None: computed_size = stream_obj.getbuffer().nbytes else: - stream_obj = stream + stream_obj = cast(BinaryIO, stream) if computed_size is None: if hasattr(stream_obj, "getbuffer"): computed_size = stream_obj.getbuffer().nbytes @@ -888,7 +891,7 @@ def _format_response_payload( content = bytes(response) if isinstance(response_type, type) and issubclass(response_type, proto.Message): - return response_type.from_json(content, ignore_unknown_fields=True) + return cast(Any, response_type).from_json(content, ignore_unknown_fields=True) if isinstance(response_type, type) and issubclass( response_type, google.protobuf.message.Message ): diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 445b75257105..7a1e2fe35af2 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -202,6 +202,16 @@ def _ensure_aiohttp(self) -> None: "Please install google-api-core[async_rest]." ) + def _enrich_exception(self, exc: BaseException) -> None: + """Attaches session diagnostic metadata to an active exception. + + Args: + exc: Exception instance to augment with upload_url and chunk_size. + """ + if hasattr(exc, "__dict__"): + setattr(exc, "upload_url", self.upload_url) + setattr(exc, "chunk_size", self.chunk_size) + def _notify_progress( self, state: common.ProgressState, queue: Optional[asyncio.Queue] = None ) -> None: @@ -457,7 +467,9 @@ async def do_http(): f"Resumable upload deadline {self._config.deadline} exceeded." ) raise exceptions.TransferStalledError( - f"Upload stalled: transfer rate remained below {rate} bytes/s for longer than {self._config.stall_timeout}s." + f"Upload stalled: transfer rate remained below {rate} bytes/s for longer than {self._config.stall_timeout}s.", + upload_url=self.upload_url, + chunk_size=self.chunk_size, ) else: self._stall_timeout_started = None @@ -472,21 +484,18 @@ async def do_http(): ) return status_code, resp_headers, resp_body except Exception as exc: - if hasattr(exc, "__dict__"): - exc.upload_url = self.upload_url - exc.chunk_size = self.chunk_size + self._enrich_exception(exc) if isinstance(exc, (asyncio.TimeoutError, exceptions.DeadlineExceeded)): remaining = self._get_deadline_remaining() if remaining is not None and remaining <= 0: raise exceptions.DeadlineExceeded( f"Resumable upload deadline {self._config.deadline} exceeded." ) from exc - stalled_err = exceptions.TransferStalledError( - f"Upload stalled: chunk transfer timed out ({exc})." - ) - stalled_err.upload_url = self.upload_url - stalled_err.chunk_size = self.chunk_size - raise stalled_err from exc + raise exceptions.TransferStalledError( + f"Upload stalled: chunk transfer timed out ({exc}).", + upload_url=self.upload_url, + chunk_size=self.chunk_size, + ) from exc is_recoverable = ( isinstance(exc, exceptions.GoogleAPICallError) @@ -556,29 +565,26 @@ async def do_query(): self._buffered_chunk = None if stream_obj is not None and hasattr(stream_obj, "seek"): if hasattr(stream_obj, "seekable") and not stream_obj.seekable(): - err = exceptions.UnseekableStreamError( - f"Stream is not seekable. Cannot recover upload to offset {received}." + raise exceptions.UnseekableStreamError( + f"Stream is not seekable. Cannot recover upload to offset {received}.", + upload_url=self.upload_url, + chunk_size=self.chunk_size, ) - err.upload_url = self.upload_url - err.chunk_size = self.chunk_size - raise err try: stream_obj.seek(self._start_stream_offset + received) return received except (OSError, AttributeError) as exc: - err = exceptions.UnseekableStreamError( - f"Failed to seek stream to offset {received}: {exc}" - ) - err.upload_url = self.upload_url - err.chunk_size = self.chunk_size - raise err from exc - - err = exceptions.UnseekableStreamError( - f"Server offset {received} precedes active buffer. Stream cannot be rewound." + raise exceptions.UnseekableStreamError( + f"Failed to seek stream to offset {received}: {exc}", + upload_url=self.upload_url, + chunk_size=self.chunk_size, + ) from exc + + raise exceptions.UnseekableStreamError( + f"Server offset {received} precedes active buffer. Stream cannot be rewound.", + upload_url=self.upload_url, + chunk_size=self.chunk_size, ) - err.upload_url = self.upload_url - err.chunk_size = self.chunk_size - raise err async def cancel(self, transport: Optional[Any] = None) -> None: """Cancels the resumable upload session asynchronously. @@ -664,9 +670,7 @@ async def _run(): progress_queue.put_nowait(_DONE_SENTINEL) return self._response except Exception as exc: - if hasattr(exc, "__dict__"): - exc.upload_url = self.upload_url - exc.chunk_size = self.chunk_size + self._enrich_exception(exc) progress_queue.put_nowait(exc) raise @@ -735,9 +739,7 @@ async def _run(): progress_queue.put_nowait(_DONE_SENTINEL) return self._response except Exception as exc: - if hasattr(exc, "__dict__"): - exc.upload_url = self.upload_url - exc.chunk_size = self.chunk_size + self._enrich_exception(exc) progress_queue.put_nowait(exc) raise @@ -764,17 +766,17 @@ def _prepare_async_reader( TypeError: If the stream type is not supported. """ computed_size = size - stream_obj = None + stream_obj: Any = None if isinstance(stream, bytes): - stream_obj = io.BytesIO(stream) + bytes_io = io.BytesIO(stream) if computed_size is None: computed_size = len(stream) async def reader(n: int) -> bytes: - return stream_obj.read(n) + return bytes_io.read(n) - return reader, computed_size, stream_obj + return reader, computed_size, bytes_io if hasattr(stream, "read") and inspect.iscoroutinefunction(stream.read): # Native async reader (e.g. asyncio.StreamReader) @@ -785,20 +787,20 @@ async def reader(n: int) -> bytes: if hasattr(stream, "read"): # Synchronous binary stream: offload blocking reads to worker thread - stream_obj = stream - if computed_size is None and hasattr(stream, "getbuffer"): - computed_size = stream.getbuffer().nbytes + sync_stream: Any = stream + if computed_size is None and hasattr(sync_stream, "getbuffer"): + computed_size = sync_stream.getbuffer().nbytes - if hasattr(stream_obj, "tell"): + if hasattr(sync_stream, "tell"): try: - self._start_stream_offset = stream_obj.tell() + self._start_stream_offset = sync_stream.tell() except (OSError, AttributeError): self._start_stream_offset = 0 async def reader(n: int) -> bytes: - return await asyncio.to_thread(stream.read, n) # type: ignore + return await asyncio.to_thread(sync_stream.read, n) - return reader, computed_size, stream_obj + return reader, computed_size, sync_stream if hasattr(stream, "__aiter__"): # Native AsyncIterable[bytes] From 9c49a8107315992eb91e0c3864b11ff745b46b7f Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 16:10:11 +0000 Subject: [PATCH 05/43] lint --- .../google/api_core/resumable_transfer/upload_async.py | 1 - .../google-api-core/tests/asyncio/test_grpc_helpers_async.py | 2 +- .../google-api-core/tests/asyncio/test_rest_streaming_async.py | 2 +- packages/google-api-core/tests/helpers.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 7a1e2fe35af2..4cf285664b3c 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -766,7 +766,6 @@ def _prepare_async_reader( TypeError: If the stream type is not supported. """ computed_size = size - stream_obj: Any = None if isinstance(stream, bytes): bytes_io = io.BytesIO(stream) diff --git a/packages/google-api-core/tests/asyncio/test_grpc_helpers_async.py b/packages/google-api-core/tests/asyncio/test_grpc_helpers_async.py index dcb09f18fea2..c90c6c7bceeb 100644 --- a/packages/google-api-core/tests/asyncio/test_grpc_helpers_async.py +++ b/packages/google-api-core/tests/asyncio/test_grpc_helpers_async.py @@ -17,7 +17,7 @@ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401 except ImportError: # pragma: NO COVER import mock # type: ignore -import pytest # noqa: I202 +import pytest from ..helpers import warn_deprecated_credentials_file diff --git a/packages/google-api-core/tests/asyncio/test_rest_streaming_async.py b/packages/google-api-core/tests/asyncio/test_rest_streaming_async.py index 743a60fb05ab..61919aac5604 100644 --- a/packages/google-api-core/tests/asyncio/test_rest_streaming_async.py +++ b/packages/google-api-core/tests/asyncio/test_rest_streaming_async.py @@ -28,7 +28,7 @@ import mock # type: ignore import proto -import pytest # noqa: I202 +import pytest try: from google.auth.aio.transport import Response diff --git a/packages/google-api-core/tests/helpers.py b/packages/google-api-core/tests/helpers.py index 86b5d149755f..cd55a32e9da5 100644 --- a/packages/google-api-core/tests/helpers.py +++ b/packages/google-api-core/tests/helpers.py @@ -19,7 +19,7 @@ from typing import List import proto -import pytest # noqa: I202 +import pytest from google.protobuf import duration_pb2, timestamp_pb2 from google.protobuf.json_format import MessageToJson From b0de58074824f8a9bd7c751857ad7e815f6e0d51 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 16:16:50 +0000 Subject: [PATCH 06/43] address review feedback --- .../api_core/resumable_transfer/upload.py | 33 ++++++++++++------ .../resumable_transfer/upload_async.py | 34 +++++++++++++------ 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index e31bdff2b682..854c7acb2588 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -301,13 +301,10 @@ def should_retry(exc: Any) -> bool: ( requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError, + requests.exceptions.Timeout, ), ): return True - if isinstance(exc, requests.exceptions.Timeout): - if self._config.stall_minimum_rate and self._config.stall_timeout: - return False - return True if isinstance(exc, exceptions.GoogleAPICallError): return exc.code in common.RETRYABLE_STATUS_CODES return False @@ -517,13 +514,27 @@ def do_transmit() -> requests.Response: def do_http() -> requests.Response: per_attempt_timeout = self._compute_chunk_timeout(data_len) - resp = transport.request( - method, - url, - data=payload, - headers=headers, - timeout=per_attempt_timeout, - ) + try: + resp = transport.request( + method, + url, + data=payload, + headers=headers, + timeout=per_attempt_timeout, + ) + except requests.exceptions.Timeout as exc: + if self._config.stall_minimum_rate and self._config.stall_timeout: + remaining = self._get_deadline_remaining() + if remaining is not None and remaining <= 0: + raise exceptions.DeadlineExceeded( + f"Resumable upload deadline {self._config.deadline} exceeded." + ) from exc + raise exceptions.TransferStalledError( + f"Upload stalled: chunk transfer timed out ({exc}).", + upload_url=self.upload_url, + chunk_size=self.chunk_size, + ) from exc + raise if not resp.ok: raise exceptions.from_http_response(resp) return resp diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 4cf285664b3c..c2938a049622 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -432,16 +432,30 @@ async def do_http(): per_attempt_timeout = min(per_attempt_timeout, remaining) client_timeout = aiohttp.ClientTimeout(total=per_attempt_timeout) - async with transport.request( - method, url, data=payload, headers=headers, timeout=client_timeout - ) as resp: - resp_headers = dict(resp.headers) - resp_body = await resp.read() - if resp.status not in (200, 201): - raise exceptions.from_http_status( - resp.status, resp_body.decode("utf-8", errors="replace") - ) - return resp.status, resp_headers, resp_body + try: + async with transport.request( + method, url, data=payload, headers=headers, timeout=client_timeout + ) as resp: + resp_headers = dict(resp.headers) + resp_body = await resp.read() + if resp.status not in (200, 201): + raise exceptions.from_http_status( + resp.status, resp_body.decode("utf-8", errors="replace") + ) + return resp.status, resp_headers, resp_body + except asyncio.TimeoutError as exc: + if self._config.stall_minimum_rate and self._config.stall_timeout: + remaining = self._get_deadline_remaining() + if remaining is not None and remaining <= 0: + raise exceptions.DeadlineExceeded( + f"Resumable upload deadline {self._config.deadline} exceeded." + ) from exc + raise exceptions.TransferStalledError( + f"Upload stalled: chunk transfer timed out ({exc}).", + upload_url=self.upload_url, + chunk_size=self.chunk_size, + ) from exc + raise try: t_start = _monotonic_clock() From ae11c273d51f999de7a1f069b737632a0e6d7a81 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 17:21:25 +0000 Subject: [PATCH 07/43] address review feedback --- .../api_core/resumable_transfer/upload.py | 6 ++++- .../resumable_transfer/upload_async.py | 15 ++++++----- .../asyncio/test_resumable_transfer_async.py | 27 +++++++++++++++++++ .../tests/unit/test_resumable_transfer.py | 24 +++++++++++++++++ 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 854c7acb2588..efef38983a9e 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -833,6 +833,8 @@ def _prepare_stream( Tuple of (prepared BinaryIO stream, computed total size). """ computed_size = size + if isinstance(stream, (str, dict)): + raise TypeError(f"Unsupported stream type: {type(stream)}") if isinstance(stream, bytes): stream_obj: BinaryIO = io.BytesIO(stream) if computed_size is None: @@ -841,7 +843,7 @@ def _prepare_stream( stream_obj = io.BytesIO(b"".join(stream)) if computed_size is None: computed_size = stream_obj.getbuffer().nbytes - else: + elif hasattr(stream, "read"): stream_obj = cast(BinaryIO, stream) if computed_size is None: if hasattr(stream_obj, "getbuffer"): @@ -855,6 +857,8 @@ def _prepare_stream( stream_obj.seek(0, io.SEEK_END) computed_size = stream_obj.tell() - cur stream_obj.seek(cur) + else: + raise TypeError(f"Unsupported stream type: {type(stream)}") if hasattr(stream_obj, "tell"): try: diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index c2938a049622..5c040c7e5aca 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -655,12 +655,12 @@ def upload( raise ValueError("An aiohttp.ClientSession transport must be provided.") progress_queue: asyncio.Queue = asyncio.Queue() + reader_fn, computed_size, stream_obj = self._prepare_async_reader( + stream, size + ) async def _run(): try: - reader_fn, computed_size, stream_obj = self._prepare_async_reader( - stream, size - ) await self.initiate( transport=sess, request_body=request_body, @@ -726,12 +726,12 @@ def resume( self._state._resumable_url = upload_url progress_queue: asyncio.Queue = asyncio.Queue() + reader_fn, computed_size, stream_obj = self._prepare_async_reader( + stream, size + ) async def _run(): try: - reader_fn, computed_size, stream_obj = self._prepare_async_reader( - stream, size - ) await self._recover(sess, stream_obj) self._notify_progress( common.ProgressState.OFFSET_RECEIVED, progress_queue @@ -781,6 +781,9 @@ def _prepare_async_reader( """ computed_size = size + if isinstance(stream, (str, dict)): + raise TypeError(f"Unsupported stream type: {type(stream)}") + if isinstance(stream, bytes): bytes_io = io.BytesIO(stream) if computed_size is None: diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 28bb290939a3..9193e882d1d4 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -984,3 +984,30 @@ async def test_async_operation_error_propagation_in_progress() -> None: with pytest.raises(exceptions.Forbidden): await upload_op + + +@pytest.mark.parametrize("invalid_stream", ["invalid_string", {"key": "value"}, 12345]) +def test_async_upload_rejects_invalid_stream_types(invalid_stream: Any) -> None: + """Verifies that str, dict, and non-stream objects raise TypeError synchronously on upload().""" + async_transport = DummyAsyncSession([]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + ) + with pytest.raises(TypeError, match="Unsupported stream type"): + session.upload(stream=invalid_stream) + + +@pytest.mark.parametrize("invalid_stream", ["invalid_string", {"key": "value"}, 12345]) +def test_async_resume_rejects_invalid_stream_types(invalid_stream: Any) -> None: + """Verifies that str, dict, and non-stream objects raise TypeError synchronously on resume().""" + async_transport = DummyAsyncSession([]) + session = AsyncResumableUploadSession( + transport=async_transport, + ) + with pytest.raises(TypeError, match="Unsupported stream type"): + session.resume( + upload_url="https://upload.example.com/resumable-async", + stream=invalid_stream, + ) + diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 20c5532a8a84..6cef77613067 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -725,3 +725,27 @@ def test_sync_response_type_raw_response(): ) resp = session.upload(stream=b"payload") assert resp is chunk_resp + + +def test_sync_retry_predicate_allows_timeout_with_stall_control(): + config = ResumableUploadConfig(stall_minimum_rate=1024, stall_timeout=1.0) + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + ) + predicate = session._get_retry_predicate() + assert predicate(requests.exceptions.Timeout("Read timed out")) is True + + + +@pytest.mark.parametrize("invalid_stream", ["invalid_string", {"key": "value"}, 12345]) +def test_sync_upload_rejects_invalid_stream_types(invalid_stream): + session_transport = mock.create_autospec(requests.Session, instance=True) + session = ResumableUploadSession( + upload_url="https://api.example.com/start", + transport=session_transport, + ) + with pytest.raises(TypeError, match="Unsupported stream type"): + session.upload(stream=invalid_stream) + + From f57c59817723392187bf843d2cc352fd46091db7 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 13:23:26 -0400 Subject: [PATCH 08/43] Update packages/google-api-core/google/api_core/resumable_transfer/upload_state.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../google/api_core/resumable_transfer/upload_state.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py index 234e3e6c9526..f748cb01e9cb 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_state.py @@ -111,7 +111,9 @@ def build_start_request( if headers: for k, v in headers: - req_headers[k] = v.decode("utf-8") if isinstance(v, bytes) else str(v) + key = k.decode("utf-8") if isinstance(k, bytes) else str(k) + val = v.decode("utf-8") if isinstance(v, bytes) else str(v) + req_headers[key] = val req_headers[common.HEADER_PROTOCOL] = common.PROTOCOL_RESUMABLE req_headers[common.HEADER_COMMAND] = common.Command.START.value From efbc62306c8f3518e5e9a37ab6f0e4419f30927b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 13:23:38 -0400 Subject: [PATCH 09/43] Update packages/google-api-core/google/api_core/resumable_transfer/upload_async.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../google/api_core/resumable_transfer/upload_async.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 5c040c7e5aca..96784ac7574a 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -475,11 +475,7 @@ async def do_http(): _monotonic_clock() - self._stall_timeout_started >= self._config.stall_timeout ): - remaining = self._get_deadline_remaining() - if remaining is not None and remaining <= 0: - raise exceptions.DeadlineExceeded( - f"Resumable upload deadline {self._config.deadline} exceeded." - ) + self._get_deadline_remaining() raise exceptions.TransferStalledError( f"Upload stalled: transfer rate remained below {rate} bytes/s for longer than {self._config.stall_timeout}s.", upload_url=self.upload_url, From 41a375952bb69bebd0862db18fae294ac4ee0622 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 13:23:53 -0400 Subject: [PATCH 10/43] Update packages/google-api-core/google/api_core/resumable_transfer/upload.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../google/api_core/resumable_transfer/upload.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index efef38983a9e..446d3c1054e5 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -381,11 +381,7 @@ def _update_stall_control( _monotonic_clock() - self._stall_timeout_started >= self._config.stall_timeout ): - remaining = self._get_deadline_remaining() - if remaining is not None and remaining <= 0: - raise exceptions.DeadlineExceeded( - f"Resumable upload deadline {self._config.deadline} exceeded." - ) + self._get_deadline_remaining() raise exceptions.TransferStalledError( f"Upload stalled: transfer rate remained below {rate} bytes/s " f"for longer than {self._config.stall_timeout}s.", From 405396ef462524c3e0faa68ecca856788007b9dc Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 17:35:02 +0000 Subject: [PATCH 11/43] lint --- .../api_core/resumable_transfer/upload_async.py | 14 +++++++------- .../tests/asyncio/test_resumable_transfer_async.py | 1 - .../tests/unit/test_resumable_transfer.py | 3 --- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 96784ac7574a..bd5616e93a12 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -434,7 +434,11 @@ async def do_http(): client_timeout = aiohttp.ClientTimeout(total=per_attempt_timeout) try: async with transport.request( - method, url, data=payload, headers=headers, timeout=client_timeout + method, + url, + data=payload, + headers=headers, + timeout=client_timeout, ) as resp: resp_headers = dict(resp.headers) resp_body = await resp.read() @@ -651,9 +655,7 @@ def upload( raise ValueError("An aiohttp.ClientSession transport must be provided.") progress_queue: asyncio.Queue = asyncio.Queue() - reader_fn, computed_size, stream_obj = self._prepare_async_reader( - stream, size - ) + reader_fn, computed_size, stream_obj = self._prepare_async_reader(stream, size) async def _run(): try: @@ -722,9 +724,7 @@ def resume( self._state._resumable_url = upload_url progress_queue: asyncio.Queue = asyncio.Queue() - reader_fn, computed_size, stream_obj = self._prepare_async_reader( - stream, size - ) + reader_fn, computed_size, stream_obj = self._prepare_async_reader(stream, size) async def _run(): try: diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 9193e882d1d4..e45a56e6adc6 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -1010,4 +1010,3 @@ def test_async_resume_rejects_invalid_stream_types(invalid_stream: Any) -> None: upload_url="https://upload.example.com/resumable-async", stream=invalid_stream, ) - diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 6cef77613067..cfebf0295386 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -737,7 +737,6 @@ def test_sync_retry_predicate_allows_timeout_with_stall_control(): assert predicate(requests.exceptions.Timeout("Read timed out")) is True - @pytest.mark.parametrize("invalid_stream", ["invalid_string", {"key": "value"}, 12345]) def test_sync_upload_rejects_invalid_stream_types(invalid_stream): session_transport = mock.create_autospec(requests.Session, instance=True) @@ -747,5 +746,3 @@ def test_sync_upload_rejects_invalid_stream_types(invalid_stream): ) with pytest.raises(TypeError, match="Unsupported stream type"): session.upload(stream=invalid_stream) - - From 5409b001a88eac5cb1577b4e4bc30d04ddd27a2a Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 17:49:31 +0000 Subject: [PATCH 12/43] additional headers->headers --- .../google/api_core/resumable_transfer/upload.py | 12 ++++++------ .../tests/unit/test_resumable_transfer.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 446d3c1054e5..16329f050036 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -65,7 +65,7 @@ class ResumableUploadConfig: start_retry: Custom retry policy for the start request. stall_minimum_rate: Minimum transfer rate in bytes per second. Defaults to 64 KiB/s. stall_timeout: Stall duration threshold in seconds. Defaults to 120s. - additional_headers: Additional HTTP headers dispatched exclusively with start request. + headers: Additional HTTP headers dispatched exclusively with start request. deadline: Overall global deadline for the upload process. timeout: Fallback per-request timeout. retry: Fallback retry policy. @@ -80,7 +80,7 @@ class ResumableUploadConfig: start_retry: Optional[google.api_core.retry.Retry] = None stall_minimum_rate: int = 64 * 1024 stall_timeout: float = 120.0 - additional_headers: Optional[ + headers: Optional[ Union[Mapping[str, str], Sequence[Tuple[str, str]]] ] = None deadline: Optional[datetime.datetime] = None @@ -105,11 +105,11 @@ def __post_init__(self) -> None: @property def start_headers(self) -> Optional[Sequence[Tuple[str, str]]]: """Returns normalized additional headers for the start request.""" - if self.additional_headers is None: + if self.headers is None: return None - if isinstance(self.additional_headers, Mapping): - return list(self.additional_headers.items()) - return list(self.additional_headers) + if isinstance(self.headers, Mapping): + return list(self.headers.items()) + return list(self.headers) class ResumableUploadSession: diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index cfebf0295386..b04bf65d852f 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -254,7 +254,7 @@ def test_resumable_upload_config_defaults(): assert config.stall_timeout == 120.0 assert config.start_timeout is None assert config.start_retry is None - assert config.additional_headers is None + assert config.headers is None assert config.deadline is None @@ -263,7 +263,7 @@ def test_resumable_upload_config_fallbacks_and_headers(): config1 = ResumableUploadConfig( start_timeout=45.0, retry=retry1, - additional_headers={"X-Test": "1"}, + headers={"X-Test": "1"}, ) assert config1.timeout == 45.0 assert config1.start_timeout == 45.0 @@ -275,7 +275,7 @@ def test_resumable_upload_config_fallbacks_and_headers(): config2 = ResumableUploadConfig( timeout=30.0, start_retry=retry2, - additional_headers=[("X-Test", "2")], + headers=[("X-Test", "2")], ) assert config2.timeout == 30.0 assert config2.start_timeout == 30.0 From 0e0ad52b357f02bcc0efb569b9f82965e82fdd74 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 17:55:50 +0000 Subject: [PATCH 13/43] lint --- .../google/api_core/resumable_transfer/upload.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 16329f050036..4b771e100302 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -80,9 +80,7 @@ class ResumableUploadConfig: start_retry: Optional[google.api_core.retry.Retry] = None stall_minimum_rate: int = 64 * 1024 stall_timeout: float = 120.0 - headers: Optional[ - Union[Mapping[str, str], Sequence[Tuple[str, str]]] - ] = None + headers: Optional[Union[Mapping[str, str], Sequence[Tuple[str, str]]]] = None deadline: Optional[datetime.datetime] = None timeout: Optional[float] = None retry: Optional[google.api_core.retry.Retry] = None From 3936458ec48a1d5fb1c04632e3deed647f2a6082 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 18:29:52 +0000 Subject: [PATCH 14/43] cover --- .../tests/unit/test_resumable_transfer.py | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index b04bf65d852f..6ca54ca37ffa 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import io from typing import Union from unittest import mock @@ -20,6 +21,7 @@ import requests from google.protobuf import empty_pb2 +import google.api_core.retry from google.api_core import exceptions from google.api_core.resumable_transfer import ( DEFAULT_CHUNK_SIZE, @@ -746,3 +748,254 @@ def test_sync_upload_rejects_invalid_stream_types(invalid_stream): ) with pytest.raises(TypeError, match="Unsupported stream type"): session.upload(stream=invalid_stream) + + +def test_upload_state_properties(): + state = upload_state.ResumableUploadState( + "https://api.example.com/init", chunk_size=500 + ) + assert state.initial_url == "https://api.example.com/init" + assert state.resumable_url is None + assert state.bytes_uploaded == 0 + assert state.total_bytes is None + assert state.finished is False + assert state.invalid is False + assert state.chunk_size == 500 + + # With granularity alignment + state._chunk_granularity = 256 + assert state.chunk_size == 512 + + +def test_upload_state_start_errors(): + state = upload_state.ResumableUploadState("https://api.example.com/init") + with pytest.raises(ValueError, match="Start command failed with status 500"): + state.process_start_response(500, {}) + assert state.invalid is True + + state2 = upload_state.ResumableUploadState("https://api.example.com/init") + with pytest.raises(ValueError, match="Server did not return"): + state2.process_start_response(200, {"X-Goog-Upload-Status": "active"}) + assert state2.invalid is True + + +def test_upload_state_chunk_and_query_errors(): + state = upload_state.ResumableUploadState("https://api.example.com/init") + with pytest.raises(ValueError, match="Upload session URL not established"): + state.build_chunk_request(b"data", is_last_chunk=True) + + with pytest.raises(ValueError, match="Upload session URL not established"): + state.build_query_request() + + with pytest.raises(ValueError, match="Upload session URL not established"): + state.build_cancel_request() + + # process_chunk_response with non-200/201 status code + state.process_chunk_response(503, {}, 10) + assert state.bytes_uploaded == 0 + + # process_query_response with non-200/201 status code + with pytest.raises(ValueError, match="Query recovery failed with status 500"): + state.process_query_response(500, {}) + assert state.invalid is True + + # process_query_response with final status + state3 = upload_state.ResumableUploadState("https://api.example.com/init") + state3.process_query_response(200, {"X-Goog-Upload-Status": "final"}) + assert state3.finished is True + + # process_query_response with cancelled status + state4 = upload_state.ResumableUploadState("https://api.example.com/init") + with pytest.raises(UploadCancelledError): + state4.process_query_response(200, {"X-Goog-Upload-Status": "cancelled"}) + assert state4.invalid is True + + +def test_sync_upload_session_properties_and_enrichment(): + session_transport = mock.create_autospec(requests.Session, instance=True) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=session_transport, + ) + assert session._ensure_session() is session_transport + assert session.resumable_url is None + assert session.bytes_uploaded == 0 + assert session.total_bytes is None + assert session.finished is False + assert session.invalid is False + + # Exception without __dict__ does not fail _enrich_exception + exc_no_dict = Exception() + session._enrich_exception(exc_no_dict) + + +def test_sync_upload_session_transport_missing(): + session = ResumableUploadSession(upload_url="https://api.example.com/init") + with pytest.raises( + ValueError, match="A requests.Session transport must be provided" + ): + session.upload(stream=b"payload") + + with pytest.raises( + ValueError, match="A requests.Session transport must be provided" + ): + session.cancel() + + +def test_sync_deadline_handling(): + past = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=10) + config = ResumableUploadConfig(deadline=past) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + ) + with pytest.raises(exceptions.DeadlineExceeded): + session._get_deadline_remaining() + + future_naive = datetime.datetime.now() + datetime.timedelta(hours=1) + config2 = ResumableUploadConfig(deadline=future_naive) + session2 = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config2, + ) + rem = session2._get_deadline_remaining() + assert rem is not None and rem > 0 + + +def test_sync_retry_predicate_branches(): + session = ResumableUploadSession(upload_url="https://api.example.com/init") + pred = session._get_retry_predicate() + + assert pred(exceptions.DeadlineExceeded("deadline")) is False + assert pred(TransferStalledError("stalled")) is False + assert pred(UploadCancelledError("cancelled")) is False + assert pred(MissingStatusHeaderError("missing")) is True + assert pred(requests.exceptions.ConnectionError("conn")) is True + assert pred(requests.exceptions.ChunkedEncodingError("chunked")) is True + assert pred(exceptions.from_http_status(503, "503")) is True + assert pred(exceptions.from_http_status(400, "400")) is False + assert pred(TypeError("other")) is False + + +def test_sync_reposition_stream_errors(): + session_transport = mock.create_autospec(requests.Session, instance=True) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=session_transport, + ) + + unseekable = mock.Mock() + unseekable.seekable.return_value = False + with pytest.raises(UnseekableStreamError, match="Stream is not seekable"): + session._reposition_stream_offset(unseekable, 100) + + failing_seek = mock.Mock() + failing_seek.seekable.return_value = True + failing_seek.seek.side_effect = OSError("Disk read failure") + with pytest.raises(UnseekableStreamError, match="Failed to seek stream"): + session._reposition_stream_offset(failing_seek, 100) + + +def test_sync_prepare_stream_seekable_and_iterable(): + session_transport = mock.create_autospec(requests.Session, instance=True) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=session_transport, + ) + + stream_obj, computed_size = session._prepare_stream([b"hello ", b"world"], None) + assert stream_obj.read() == b"hello world" + assert computed_size == 11 + + class CustomSeekable: + def __init__(self, data: bytes): + self._bio = io.BytesIO(data) + + def read(self, n: int = -1) -> bytes: + return self._bio.read(n) + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + return self._bio.seek(offset, whence) + + def tell(self) -> int: + return self._bio.tell() + + def seekable(self) -> bool: + return True + + custom = CustomSeekable(b"0123456789") + custom.seek(2) + stream_obj2, computed_size2 = session._prepare_stream(custom, None) + assert computed_size2 == 8 + assert custom.tell() == 2 + + +def test_sync_config_fallbacks_and_headers(): + cfg1 = ResumableUploadConfig(start_timeout=15.0) + assert cfg1.timeout == 15.0 + + cfg2 = ResumableUploadConfig(timeout=25.0) + assert cfg2.start_timeout == 25.0 + + ret = mock.Mock(spec=google.api_core.retry.Retry) + cfg3 = ResumableUploadConfig(start_retry=ret) + assert cfg3.retry is ret + + cfg4 = ResumableUploadConfig(retry=ret) + assert cfg4.start_retry is ret + + cfg_dict = ResumableUploadConfig(headers={"X-Key": "Val"}) + assert cfg_dict.start_headers == [("X-Key", "Val")] + + cfg_list = ResumableUploadConfig(headers=[("X-Key", "Val")]) + assert cfg_list.start_headers == [("X-Key", "Val")] + + cfg_none = ResumableUploadConfig(headers=None) + assert cfg_none.start_headers is None + + +def test_sync_cancel_failure_raises(): + session_transport = mock.create_autospec(requests.Session, instance=True) + err_resp = mock.create_autospec(requests.Response, instance=True) + err_resp.ok = False + err_resp.status_code = 500 + err_resp.headers = {} + err_resp.json.return_value = {"error": {"message": "Server Error", "errors": []}} + session_transport.request.return_value = err_resp + + session = ResumableUploadSession( + resumable_url="https://upload.example.com/resumable-123", + transport=session_transport, + ) + with pytest.raises(exceptions.GoogleAPICallError): + session.cancel() + + +def test_sync_resume_chunk_size_override(): + session_transport = mock.create_autospec(requests.Session, instance=True) + query_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "0", + }, + ) + chunk_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b"done", + ) + session_transport.request.side_effect = [query_resp, chunk_resp] + + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=session_transport, + ) + session.resume( + upload_url="https://upload.example.com/resumable-123", + stream=b"data", + chunk_size=1024, + ) + assert session.chunk_size == 1024 From 8c9c900fddcdb12780566997b89f207231d12e28 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 18:34:05 +0000 Subject: [PATCH 15/43] cover --- .../tests/unit/test_resumable_transfer.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 6ca54ca37ffa..c7248cd448ce 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -751,9 +751,7 @@ def test_sync_upload_rejects_invalid_stream_types(invalid_stream): def test_upload_state_properties(): - state = upload_state.ResumableUploadState( - "https://api.example.com/init", chunk_size=500 - ) + state = upload_state.ProtocolState("https://api.example.com/init", chunk_size=500) assert state.initial_url == "https://api.example.com/init" assert state.resumable_url is None assert state.bytes_uploaded == 0 @@ -768,19 +766,19 @@ def test_upload_state_properties(): def test_upload_state_start_errors(): - state = upload_state.ResumableUploadState("https://api.example.com/init") + state = upload_state.ProtocolState("https://api.example.com/init") with pytest.raises(ValueError, match="Start command failed with status 500"): state.process_start_response(500, {}) assert state.invalid is True - state2 = upload_state.ResumableUploadState("https://api.example.com/init") + state2 = upload_state.ProtocolState("https://api.example.com/init") with pytest.raises(ValueError, match="Server did not return"): state2.process_start_response(200, {"X-Goog-Upload-Status": "active"}) assert state2.invalid is True def test_upload_state_chunk_and_query_errors(): - state = upload_state.ResumableUploadState("https://api.example.com/init") + state = upload_state.ProtocolState("https://api.example.com/init") with pytest.raises(ValueError, match="Upload session URL not established"): state.build_chunk_request(b"data", is_last_chunk=True) @@ -800,12 +798,12 @@ def test_upload_state_chunk_and_query_errors(): assert state.invalid is True # process_query_response with final status - state3 = upload_state.ResumableUploadState("https://api.example.com/init") + state3 = upload_state.ProtocolState("https://api.example.com/init") state3.process_query_response(200, {"X-Goog-Upload-Status": "final"}) assert state3.finished is True # process_query_response with cancelled status - state4 = upload_state.ResumableUploadState("https://api.example.com/init") + state4 = upload_state.ProtocolState("https://api.example.com/init") with pytest.raises(UploadCancelledError): state4.process_query_response(200, {"X-Goog-Upload-Status": "cancelled"}) assert state4.invalid is True @@ -817,12 +815,12 @@ def test_sync_upload_session_properties_and_enrichment(): upload_url="https://api.example.com/init", transport=session_transport, ) - assert session._ensure_session() is session_transport - assert session.resumable_url is None + assert session._get_transport(None) is session_transport + assert session._state.resumable_url is None assert session.bytes_uploaded == 0 - assert session.total_bytes is None + assert session._state.total_bytes is None assert session.finished is False - assert session.invalid is False + assert session._state.invalid is False # Exception without __dict__ does not fail _enrich_exception exc_no_dict = Exception() @@ -960,6 +958,7 @@ def test_sync_cancel_failure_raises(): err_resp.ok = False err_resp.status_code = 500 err_resp.headers = {} + err_resp.request = mock.Mock(method="POST", url="https://upload.example.com") err_resp.json.return_value = {"error": {"message": "Server Error", "errors": []}} session_transport.request.return_value = err_resp From 96f31144fe5be2d4bd28ce3b8fb9e95c1991e9f1 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 18:44:35 +0000 Subject: [PATCH 16/43] cover --- .../api_core/resumable_transfer/upload.py | 34 +- .../tests/unit/test_resumable_transfer.py | 297 ++++++++++++++++++ 2 files changed, 312 insertions(+), 19 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 4b771e100302..abd3be2324b8 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -145,6 +145,7 @@ def __init__( # Stall control tracking via monotonic clock self._aggregate_lag: float = 0.0 self._stall_timeout_started: Optional[float] = None + self._captured_progress: Optional[List[common.UploadProgress]] = None @property def upload_url(self) -> Optional[str]: @@ -199,21 +200,23 @@ def _enrich_exception(self, exc: BaseException) -> None: setattr(exc, "chunk_size", self.chunk_size) def _notify_progress(self, state: common.ProgressState) -> None: - """Notifies registered progress callback with current upload status. + """Notifies progress with current upload status. Args: state: ProgressState transition milestone. """ - if self._config.on_progress and self.upload_url: - self._config.on_progress( - common.UploadProgress( - upload_url=self.upload_url, - chunk_size=self.chunk_size, - bytes_uploaded=self._state.bytes_uploaded, - total_bytes=self._state.total_bytes, - state=state, - ) + if self.upload_url: + progress = common.UploadProgress( + upload_url=self.upload_url, + chunk_size=self.chunk_size, + bytes_uploaded=self._state.bytes_uploaded, + total_bytes=self._state.total_bytes, + state=state, ) + if self._captured_progress is not None: + self._captured_progress.append(progress) + if self._config.on_progress: + self._config.on_progress(progress) @contextlib.contextmanager def _capture_progress( @@ -225,18 +228,11 @@ def _capture_progress( List buffering UploadProgress snapshots during generator execution. """ captured: List[common.UploadProgress] = [] - old_cb = self._config.on_progress - - def capture(p: common.UploadProgress) -> None: - captured.append(p) - if old_cb: - old_cb(p) - - self._config.on_progress = capture + self._captured_progress = captured try: yield captured finally: - self._config.on_progress = old_cb + self._captured_progress = None def _get_deadline_remaining(self) -> Optional[float]: """Calculates remaining seconds until the configured upload deadline. diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index c7248cd448ce..5af3e9e9af33 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -998,3 +998,300 @@ def test_sync_resume_chunk_size_override(): chunk_size=1024, ) assert session.chunk_size == 1024 + + +def test_sync_on_progress_and_capture(): + callback_mock = mock.Mock() + config = ResumableUploadConfig(on_progress=callback_mock) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + ) + session._state._resumable_url = "https://api.example.com/init" + with session._capture_progress() as captured: + session._notify_progress(common.ProgressState.UPLOADING) + assert len(captured) == 1 + assert captured[0].state == common.ProgressState.UPLOADING + assert callback_mock.called + assert callback_mock.call_args[0][0] is captured[0] + + +def test_sync_naive_deadline_tz(): + naive = datetime.datetime.now() + datetime.timedelta(hours=1) + config = ResumableUploadConfig(deadline=naive) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + ) + rem = session._get_deadline_remaining() + assert rem is not None and rem > 0 + assert session._get_start_timeout() <= rem + + +def test_sync_get_retry_start_and_fallback(): + ret_start = mock.Mock(spec=google.api_core.retry.Retry) + ret_fallback = mock.Mock(spec=google.api_core.retry.Retry) + config = ResumableUploadConfig(start_retry=ret_start, retry=ret_fallback) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + ) + assert session._get_retry(is_start=True) is ret_start + assert session._get_retry() is ret_fallback + + +def test_sync_stall_control_with_deadline(): + import time + + # 1. compute_chunk_timeout with fallback timeout and stall control active + config = ResumableUploadConfig( + stall_minimum_rate=1024, + stall_timeout=10.0, + timeout=15.0, + ) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + ) + t1 = session._compute_chunk_timeout(512) + assert t1 <= 15.0 + + # 2. compute_chunk_timeout with deadline active + config2 = ResumableUploadConfig( + stall_minimum_rate=1024, + stall_timeout=10.0, + deadline=datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=5), + ) + session2 = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config2, + ) + t2 = session2._compute_chunk_timeout(512) + assert t2 <= 5.0 + + # 3. update_stall_control raises DeadlineExceeded + config3 = ResumableUploadConfig( + stall_minimum_rate=1024, + stall_timeout=10.0, + deadline=datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(seconds=5), + ) + session3 = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config3, + ) + with pytest.raises(exceptions.DeadlineExceeded): + session3._update_stall_control(512, time.monotonic() - 15.0, 15.0) + + # 4. update_stall_control raises TransferStalledError (no deadline) + config4 = ResumableUploadConfig( + stall_minimum_rate=1024, + stall_timeout=10.0, + ) + session4 = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config4, + ) + session4._state._resumable_url = "https://api.example.com/init" + with pytest.raises(exceptions.TransferStalledError): + session4._update_stall_control(512, time.monotonic() - 15.0, 15.0) + + +def test_sync_update_stall_control_disabled(): + import time + + config = ResumableUploadConfig(stall_minimum_rate=0, stall_timeout=10.0) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + ) + session._update_stall_control(512, time.monotonic(), 5.0) + assert session._aggregate_lag == 0.0 + + +def test_sync_initiate_failure(): + transport = mock.create_autospec(requests.Session, instance=True) + resp = mock.create_autospec(requests.Response, instance=True) + resp.ok = False + resp.status_code = 400 + resp.headers = {} + resp.json.return_value = {"error": {"message": "Init Failed"}} + resp.request = mock.Mock(method="POST", url="https://api.example.com/init") + transport.request.return_value = resp + + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=transport, + ) + with pytest.raises(exceptions.GoogleAPICallError): + session.initiate(transport=transport) + + +def test_sync_transmit_empty_stream(): + transport = mock.create_autospec(requests.Session, instance=True) + resp = mock.create_autospec(requests.Response, instance=True) + resp.ok = True + resp.status_code = 200 + resp.headers = {"X-Goog-Upload-Status": "final"} + resp.content = b"done" + transport.request.return_value = resp + + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=transport, + ) + stream = io.BytesIO(b"") + session._state._resumable_url = "https://upload.example.com/resumable-123" + result = session._transmit_chunk(transport, stream, size=0) + assert result is resp + + +def test_sync_transmit_chunk_timeout_with_stall_control_active(): + transport = mock.create_autospec(requests.Session, instance=True) + transport.request.side_effect = requests.exceptions.Timeout("Read timeout") + + config = ResumableUploadConfig( + stall_minimum_rate=1024, + stall_timeout=10.0, + ) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + transport=transport, + ) + session._state._resumable_url = "https://upload.example.com/resumable-123" + + with pytest.raises(exceptions.TransferStalledError): + session._transmit_chunk(transport, io.BytesIO(b"data"), size=4) + + config_dl = ResumableUploadConfig( + stall_minimum_rate=1024, + stall_timeout=10.0, + deadline=datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=5), + ) + session_dl = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config_dl, + transport=transport, + ) + session_dl._state._resumable_url = "https://upload.example.com/resumable-123" + session_dl._get_deadline_remaining = mock.Mock(side_effect=[5.0, -1.0]) + with pytest.raises(exceptions.DeadlineExceeded): + session_dl._transmit_chunk(transport, io.BytesIO(b"data"), size=4) + + +def test_sync_transmit_chunk_timeout_outer_exception(): + transport = mock.create_autospec(requests.Session, instance=True) + transport.request.side_effect = requests.exceptions.Timeout("Read timeout") + + config = ResumableUploadConfig( + stall_minimum_rate=0, + retry=google.api_core.retry.Retry(predicate=lambda e: False), + ) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + transport=transport, + ) + session._state._resumable_url = "https://upload.example.com/resumable-123" + with pytest.raises(exceptions.TransferStalledError): + session._transmit_chunk(transport, io.BytesIO(b"data"), size=4) + + # To hit line 554-558 (outer exception handler with elapsed deadline) + config_dl = ResumableUploadConfig( + stall_minimum_rate=0, + deadline=datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=5), + retry=google.api_core.retry.Retry(predicate=lambda e: False), + ) + session_dl = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config_dl, + transport=transport, + ) + session_dl._state._resumable_url = "https://upload.example.com/resumable-123" + session_dl._get_deadline_remaining = mock.Mock(side_effect=[5.0, -1.0]) + with pytest.raises(exceptions.DeadlineExceeded): + session_dl._transmit_chunk(transport, io.BytesIO(b"data"), size=4) + + +def test_sync_recover_failure(): + transport = mock.create_autospec(requests.Session, instance=True) + resp = mock.create_autospec(requests.Response, instance=True) + resp.ok = False + resp.status_code = 400 + resp.headers = {} + resp.json.return_value = {"error": {"message": "Recovery Failed"}} + resp.request = mock.Mock( + method="POST", url="https://upload.example.com/resumable-123" + ) + transport.request.return_value = resp + + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=transport, + ) + session._state._resumable_url = "https://upload.example.com/resumable-123" + with pytest.raises(exceptions.GoogleAPICallError): + session._recover(transport, io.BytesIO(b"data")) + + +def test_sync_transmit_all_chunks_completed_without_response(): + transport = mock.create_autospec(requests.Session, instance=True) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=transport, + ) + session._state._finished = True + with pytest.raises( + ValueError, match="Upload completed without receiving a final response" + ): + list(session._transmit_all_chunks(transport, io.BytesIO(b"data"), 4)) + + +def test_sync_iter_resume_errors(): + transport = mock.create_autospec(requests.Session, instance=True) + session = ResumableUploadSession(transport=transport) + with pytest.raises(ValueError, match="An upload URL must be provided to resume"): + list(session.iter_resume(upload_url=None, stream=b"data")) + + with pytest.raises( + ValueError, match="A data stream or payload must be provided to resume" + ): + list( + session.iter_resume(upload_url="https://api.example.com/init", stream=None) + ) + + +def test_sync_prepare_stream_tell_error(): + class TellFailingStream(io.BytesIO): + def tell(self) -> int: + raise OSError("Tell failed") + + session = ResumableUploadSession() + stream = TellFailingStream(b"data") + stream_obj, computed_size = session._prepare_stream(stream, None) + assert session._start_stream_offset == 0 + + +def test_sync_format_response_payload_custom_inputs(): + from google.api_core.resumable_transfer.upload import _format_response_payload + + class CustomBytesConvertible: + def __bytes__(self) -> bytes: + return b"custom_bytes" + + res = _format_response_payload(CustomBytesConvertible(), response_type=None) + assert isinstance(res, CustomBytesConvertible) + + res_parsed = _format_response_payload( + CustomBytesConvertible(), response_type=lambda x: x + b"_extra" + ) + assert res_parsed == b"custom_bytes_extra" + + from google.protobuf import empty_pb2 + + msg_instance = empty_pb2.Empty() + res_msg = _format_response_payload(b"{}", response_type=msg_instance) + assert isinstance(res_msg, empty_pb2.Empty) From 59de7dbdf35a5ff88266a7615a6cf8d48028f006 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 18:47:33 +0000 Subject: [PATCH 17/43] lint --- packages/google-api-core/tests/unit/test_resumable_transfer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 5af3e9e9af33..4f5f9be5944b 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -1202,7 +1202,8 @@ def test_sync_transmit_chunk_timeout_outer_exception(): # To hit line 554-558 (outer exception handler with elapsed deadline) config_dl = ResumableUploadConfig( stall_minimum_rate=0, - deadline=datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=5), + deadline=datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=5), retry=google.api_core.retry.Retry(predicate=lambda e: False), ) session_dl = ResumableUploadSession( From e13ff31fc8dc8982ac106a51ffada18e15c90dbf Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 19:01:14 +0000 Subject: [PATCH 18/43] cover --- .../api_core/resumable_transfer/upload.py | 6 +- .../resumable_transfer/upload_async.py | 10 +- .../asyncio/test_resumable_transfer_async.py | 214 ++++++++++++++++++ 3 files changed, 226 insertions(+), 4 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index abd3be2324b8..39089d40eb17 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -547,9 +547,9 @@ def do_http() -> requests.Response: return resp except Exception as exc: self._enrich_exception(exc) - if isinstance( - exc, (requests.exceptions.Timeout, exceptions.DeadlineExceeded) - ): + if isinstance(exc, exceptions.DeadlineExceeded): + raise + if isinstance(exc, requests.exceptions.Timeout): remaining = self._get_deadline_remaining() if remaining is not None and remaining <= 0: raise exceptions.DeadlineExceeded( diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index bd5616e93a12..cba5ad3e5ca9 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -499,7 +499,15 @@ async def do_http(): return status_code, resp_headers, resp_body except Exception as exc: self._enrich_exception(exc) - if isinstance(exc, (asyncio.TimeoutError, exceptions.DeadlineExceeded)): + if isinstance(exc, exceptions.DeadlineExceeded): + raise + if isinstance( + exc, + ( + asyncio.TimeoutError, + aiohttp.ServerTimeoutError if aiohttp else (), + ), + ): remaining = self._get_deadline_remaining() if remaining is not None and remaining <= 0: raise exceptions.DeadlineExceeded( diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index e45a56e6adc6..69847fbf68a2 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -1010,3 +1010,217 @@ def test_async_resume_rejects_invalid_stream_types(invalid_stream: Any) -> None: upload_url="https://upload.example.com/resumable-async", stream=invalid_stream, ) + + +def test_async_enrich_exception_without_dict() -> None: + """Verifies that _enrich_exception handles objects without __dict__.""" + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + ) + exc = Exception() + session._enrich_exception(exc) + + +def test_async_notify_progress_branches() -> None: + """Verifies progress notification callbacks and queues.""" + called = [] + config = ResumableUploadConfig(on_progress=lambda p: called.append(p)) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + ) + # When upload_url is established on state + session._state._resumable_url = "https://upload.example.com/resumable-async" + q: asyncio.Queue = asyncio.Queue() + session._notify_progress(common.ProgressState.UPLOADING, queue=q) + assert len(called) == 1 + assert q.qsize() == 1 + + +def test_async_deadline_handling_and_start_timeout() -> None: + """Verifies deadline remaining calculations and start timeout calculation.""" + # Past deadline raises DeadlineExceeded + past = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=10) + config = ResumableUploadConfig(deadline=past) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + ) + with pytest.raises(exceptions.DeadlineExceeded): + session._get_deadline_remaining() + + # Naive future deadline is localized to UTC + future_naive = datetime.datetime.now() + datetime.timedelta(hours=1) + config2 = ResumableUploadConfig(deadline=future_naive) + session2 = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config2, + ) + rem = session2._get_deadline_remaining() + assert rem is not None and rem > 0 + t = session2._get_start_timeout() + assert t > 0 + + +@pytest.mark.asyncio +async def test_async_retry_branches() -> None: + """Verifies retry predicate branches in _async_retry.""" + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + ) + + # MissingStatusHeaderError retries and raises on final attempt + attempts = 0 + + async def fail_missing_header(): + nonlocal attempts + attempts += 1 + raise exceptions.MissingStatusHeaderError("missing") + + with pytest.raises(exceptions.MissingStatusHeaderError): + await session._async_retry(fail_missing_header, max_attempts=2) + assert attempts == 2 + + # Non-retryable GoogleAPICallError raises immediately + async def fail_400(): + raise exceptions.from_http_status(400, "Bad Request") + + with pytest.raises(exceptions.BadRequest): + await session._async_retry(fail_400, max_attempts=3) + + # Retryable GoogleAPICallError retries and raises on final attempt + attempts_503 = 0 + + async def fail_503(): + nonlocal attempts_503 + attempts_503 += 1 + raise exceptions.from_http_status(503, "Service Unavailable") + + with pytest.raises(exceptions.ServiceUnavailable): + await session._async_retry(fail_503, max_attempts=2) + assert attempts_503 == 2 + + +def test_async_transport_missing_errors() -> None: + """Verifies ValueError when transport is missing from upload, resume, and cancel.""" + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + ) + with pytest.raises( + ValueError, match="An aiohttp.ClientSession transport must be provided" + ): + session.upload(stream=b"data") + + with pytest.raises( + ValueError, match="An aiohttp.ClientSession transport must be provided" + ): + session.resume(upload_url="https://upload.example.com/123", stream=b"data") + + +@pytest.mark.asyncio +async def test_async_cancel_missing_transport_and_error() -> None: + """Verifies cancel method with missing transport and server error.""" + session = AsyncResumableUploadSession( + resumable_url="https://upload.example.com/123", + ) + with pytest.raises( + ValueError, match="An aiohttp.ClientSession transport must be provided" + ): + await session.cancel() + + err_resp = DummyAsyncResponse(status=500, headers={}, body=b"Cancel Error") + sess_transport = DummyAsyncSession([err_resp]) + session2 = AsyncResumableUploadSession( + resumable_url="https://upload.example.com/123", + transport=sess_transport, + ) + with pytest.raises(exceptions.GoogleAPICallError): + await session2.cancel() + + +@pytest.mark.asyncio +async def test_async_prepare_async_reader_types() -> None: + """Verifies async reader preparation for native async reader, tell error, and iterables.""" + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + ) + + # Native async reader with coroutine read + class AsyncReader: + async def read(self, n: int) -> bytes: + return b"chunk" + + reader_fn, size, obj = session._prepare_async_reader(AsyncReader(), None) + chunk = await reader_fn(5) + assert chunk == b"chunk" + + # Sync stream whose tell() raises OSError + class TellFailingStream(io.BytesIO): + def tell(self) -> int: + raise OSError("tell error") + + stream = TellFailingStream(b"data") + reader_fn2, size2, obj2 = session._prepare_async_reader(stream, None) + chunk2 = await reader_fn2(4) + assert chunk2 == b"data" + assert session._start_stream_offset == 0 + + # Sync Iterable[bytes] + reader_fn3, size3, obj3 = session._prepare_async_reader([b"part1", b"part2"], None) + chunk3 = await reader_fn3(10) + assert chunk3 == b"part1part2" + + +@pytest.mark.asyncio +async def test_async_initiate_and_recover_failures() -> None: + """Verifies initiate and recover error handling when server returns error codes.""" + err_resp = DummyAsyncResponse(status=400, headers={}, body=b"Bad Request") + sess_transport = DummyAsyncSession([err_resp]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=sess_transport, + ) + with pytest.raises(exceptions.BadRequest): + await session.initiate(transport=sess_transport) + + err_resp2 = DummyAsyncResponse(status=400, headers={}, body=b"Query Failed") + sess_transport2 = DummyAsyncSession([err_resp2]) + session2 = AsyncResumableUploadSession( + transport=sess_transport2, + ) + session2._state._resumable_url = "https://upload.example.com/123" + with pytest.raises(exceptions.BadRequest): + await session2._recover(sess_transport2) + + +@pytest.mark.asyncio +async def test_async_recover_stream_errors() -> None: + """Verifies UnseekableStreamError during async recovery.""" + query_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "active", "X-Goog-Upload-Size-Received": "10"}, + body=b"", + ) + sess_transport = DummyAsyncSession([query_resp]) + session = AsyncResumableUploadSession( + transport=sess_transport, + ) + session._state._resumable_url = "https://upload.example.com/123" + + # Stream whose seekable() returns False + unseekable = mock.Mock() + unseekable.seekable.return_value = False + with pytest.raises(UnseekableStreamError, match="Stream is not seekable"): + await session._recover(sess_transport, stream_obj=unseekable) + + # Stream whose seek() raises OSError + sess_transport2 = DummyAsyncSession([query_resp]) + session2 = AsyncResumableUploadSession( + transport=sess_transport2, + ) + session2._state._resumable_url = "https://upload.example.com/123" + failing_seek = mock.Mock() + failing_seek.seekable.return_value = True + failing_seek.seek.side_effect = OSError("Seek error") + with pytest.raises(UnseekableStreamError, match="Failed to seek stream"): + await session2._recover(sess_transport2, stream_obj=failing_seek) From c9fbb4649161576d8e50d00f2c8cff0476c45315 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 19:03:37 +0000 Subject: [PATCH 19/43] lint --- .../tests/asyncio/test_resumable_transfer_async.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 69847fbf68a2..73f13d11b15c 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -14,6 +14,7 @@ """Asynchronous tests for Resumable Upload protocol implementation.""" +import asyncio import datetime import io import json From 5bd4bec808a1d7cc65c9281078b36b7d76f3976e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 14 Sep 2026 19:27:56 +0000 Subject: [PATCH 20/43] mypy --- .../asyncio/test_resumable_transfer_async.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 73f13d11b15c..606b928d540b 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -18,7 +18,17 @@ import datetime import io import json -from typing import Any, AsyncIterator, Dict, List, Mapping, Optional, Tuple, Union +from typing import ( + Any, + AsyncIterable, + AsyncIterator, + Dict, + List, + Mapping, + Optional, + Tuple, + Union, +) from unittest import mock import pytest @@ -1146,11 +1156,13 @@ async def test_async_prepare_async_reader_types() -> None: upload_url="https://api.example.com/start", ) - # Native async reader with coroutine read - class AsyncReader: + class AsyncReader(AsyncIterable[bytes]): # Inherit to satisfy mypy async def read(self, n: int) -> bytes: return b"chunk" + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b"chunk" + reader_fn, size, obj = session._prepare_async_reader(AsyncReader(), None) chunk = await reader_fn(5) assert chunk == b"chunk" From 1e12de7338ffe6c6f08bccffa5bccf81418c1cfe Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Sep 2026 16:11:25 +0000 Subject: [PATCH 21/43] cover --- .../asyncio/test_resumable_transfer_async.py | 380 +++++++++++++++++- .../tests/unit/test_resumable_transfer.py | 261 ++++++++++++ 2 files changed, 639 insertions(+), 2 deletions(-) diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 606b928d540b..b4d1b7709b1a 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -1022,13 +1022,19 @@ def test_async_resume_rejects_invalid_stream_types(invalid_stream: Any) -> None: stream=invalid_stream, ) +# CustomNoDictObj defines __slots__ as empty and has no parent class. +# It is used to test exception/metadata enrichment when the passed object +# does not have a __dict__ attribute (such as certain system exceptions). +class CustomNoDictObj: + __slots__ = () + def test_async_enrich_exception_without_dict() -> None: """Verifies that _enrich_exception handles objects without __dict__.""" session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", ) - exc = Exception() + exc = CustomNoDictObj() session._enrich_exception(exc) @@ -1040,11 +1046,19 @@ def test_async_notify_progress_branches() -> None: upload_url="https://api.example.com/start", config=config, ) + # When upload_url is None + session._notify_progress(common.ProgressState.UPLOADING) + assert len(called) == 0 + # When upload_url is established on state session._state._resumable_url = "https://upload.example.com/resumable-async" + # Call with queue=None + session._notify_progress(common.ProgressState.UPLOADING, queue=None) + assert len(called) == 1 + q: asyncio.Queue = asyncio.Queue() session._notify_progress(common.ProgressState.UPLOADING, queue=q) - assert len(called) == 1 + assert len(called) == 2 assert q.qsize() == 1 @@ -1111,6 +1125,29 @@ async def fail_503(): await session._async_retry(fail_503, max_attempts=2) assert attempts_503 == 2 + # DeadlineExceeded propagates immediately (no retry) + async def fail_deadline(): + raise exceptions.DeadlineExceeded("deadline") + + with pytest.raises(exceptions.DeadlineExceeded): + await session._async_retry(fail_deadline, max_attempts=3) + + # General Exception propagates on last attempt + attempts_runtime = 0 + + async def fail_runtime(): + nonlocal attempts_runtime + attempts_runtime += 1 + raise RuntimeError("runtime") + + with pytest.raises(RuntimeError): + await session._async_retry(fail_runtime, max_attempts=2) + assert attempts_runtime == 2 + + # max_attempts=0 covers loop bypass falling through + res = await session._async_retry(lambda: 123, max_attempts=0) + assert res is None + def test_async_transport_missing_errors() -> None: """Verifies ValueError when transport is missing from upload, resume, and cancel.""" @@ -1237,3 +1274,342 @@ async def test_async_recover_stream_errors() -> None: failing_seek.seek.side_effect = OSError("Seek error") with pytest.raises(UnseekableStreamError, match="Failed to seek stream"): await session2._recover(sess_transport2, stream_obj=failing_seek) + + +@pytest.mark.asyncio +async def test_async_upload_with_timeout_and_deadline() -> None: + future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=60) + config = ResumableUploadConfig(timeout=30.0, deadline=future_deadline) + + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + resp = DummyAsyncResponse(status=200, headers={"X-Goog-Upload-Status": "final"}, body=b"{}") + sess_transport = DummyAsyncSession([start_resp, resp]) + + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=sess_transport, + ) + session._state._resumable_url = "https://upload.example.com/resumable-async" + + # Run the upload + res = await session.upload(stream=b"data") + assert res == b"{}" + + +@pytest.mark.asyncio +async def test_async_transmit_chunk_timeout_errors() -> None: + # 1. TimeoutError raises TransferStalledError when remaining is > 0 + future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=60) + config = ResumableUploadConfig(deadline=future_deadline) + + class TimeoutAsyncSession: + def __init__(self): + self.calls = 0 + + def request(self, *args, **kwargs): + self.calls += 1 + if self.calls == 1: + class StartContext: + async def __aenter__(self): + class Resp: + status = 200 + headers = { + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + } + async def read(self): + return b"" + return Resp() + async def __aexit__(self, exc_type, exc, tb): + pass + return StartContext() + else: + class TimeoutContext: + async def __aenter__(self): + raise asyncio.TimeoutError("timeout") + async def __aexit__(self, exc_type, exc, tb): + pass + return TimeoutContext() + + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=TimeoutAsyncSession(), + ) + session._state._resumable_url = "https://upload.example.com/resumable-async" + + with pytest.raises(exceptions.TransferStalledError): + await session.upload(stream=b"data") + + # 2. TimeoutError raises DeadlineExceeded when remaining <= 0 + past_deadline = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=10) + config2 = ResumableUploadConfig(deadline=past_deadline) + + session2 = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config2, + transport=TimeoutAsyncSession(), + ) + session2._state._resumable_url = "https://upload.example.com/resumable-async" + session2._get_deadline_remaining = lambda: -5.0 + + with pytest.raises(exceptions.DeadlineExceeded): + await session2.upload(stream=b"data") + + +@pytest.mark.asyncio +async def test_async_upload_multiple_chunks_async_iterable() -> None: + # chunk_size = 3, payload = b"012345" (6 bytes) + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk1_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "active"}, + body=b"", + ) + chunk2_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b"{}", + ) + sess_transport = DummyAsyncSession([start_resp, chunk1_resp, chunk2_resp]) + + async def async_gen(): + yield b"012" + yield b"345" + + config = ResumableUploadConfig(chunk_size=3) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=sess_transport, + ) + session._state._resumable_url = "https://upload.example.com/resumable-async" + + res = await session.upload(stream=async_gen()) + assert res == b"{}" + + +@pytest.mark.asyncio +async def test_async_prepare_async_reader_additional_branches() -> None: + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + ) + + class LocalNoTellStream: + def read(self, n): + return b"" + + class LocalCustomReadStream: + def __init__(self, data): + self.data = data + def read(self, n): + return self.data + + # 1. bytes stream with explicit size + reader_fn1, size1, obj1 = session._prepare_async_reader(b"data", size=4) + assert size1 == 4 + + # 2. NoTellStream: read but no tell + stream2 = LocalNoTellStream() + reader_fn2, size2, obj2 = session._prepare_async_reader(stream2, size=None) + assert size2 is None + + # 3. CustomReadStream: read, but no getbuffer and no tell + stream3 = LocalCustomReadStream(b"hello") + reader_fn3, size3, obj3 = session._prepare_async_reader(stream3, size=None) + assert size3 is None + + +@pytest.mark.asyncio +async def test_async_upload_empty_stream() -> None: + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b"{}", + ) + async_transport = DummyAsyncSession([start_resp, chunk_resp]) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=async_transport, + ) + res = await session.upload(stream=b"") + assert res == b"{}" + + +@pytest.mark.asyncio +async def test_async_upload_no_stall_config() -> None: + config = ResumableUploadConfig(stall_minimum_rate=0, stall_timeout=0) + + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + resp = DummyAsyncResponse(status=200, headers={"X-Goog-Upload-Status": "final"}, body=b"{}") + sess_transport = DummyAsyncSession([start_resp, resp]) + + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=sess_transport, + ) + res = await session.upload(stream=b"data") + assert res == b"{}" + + +@pytest.mark.asyncio +async def test_async_transmit_chunk_timeout_errors_no_stall() -> None: + # 1. TimeoutError raises TransferStalledError when remaining is > 0 and no stall control + future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=60) + config = ResumableUploadConfig(deadline=future_deadline, stall_minimum_rate=0, stall_timeout=0) + + class TimeoutAsyncSession: + def __init__(self): + self.calls = 0 + + def request(self, *args, **kwargs): + self.calls += 1 + if self.calls == 1: + class StartContext: + async def __aenter__(self): + class Resp: + status = 200 + headers = { + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + } + async def read(self): + return b"" + return Resp() + async def __aexit__(self, exc_type, exc, tb): + pass + return StartContext() + else: + class TimeoutContext: + async def __aenter__(self): + raise asyncio.TimeoutError("timeout") + async def __aexit__(self, exc_type, exc, tb): + pass + return TimeoutContext() + + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=TimeoutAsyncSession(), + ) + session._state._resumable_url = "https://upload.example.com/resumable-async" + + with pytest.raises(exceptions.TransferStalledError): + await session.upload(stream=b"data") + + # 2. TimeoutError raises DeadlineExceeded when remaining <= 0 and no stall control + past_deadline = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=10) + config2 = ResumableUploadConfig(deadline=past_deadline, stall_minimum_rate=0, stall_timeout=0) + + session2 = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config2, + transport=TimeoutAsyncSession(), + ) + session2._state._resumable_url = "https://upload.example.com/resumable-async" + session2._get_deadline_remaining = lambda: -5.0 + + with pytest.raises(exceptions.DeadlineExceeded): + await session2.upload(stream=b"data") + + +@pytest.mark.asyncio +async def test_async_recover_buffered_chunk_out_of_bounds() -> None: + session = AsyncResumableUploadSession() + session._state._resumable_url = "https://upload.example.com/resumable-async" + session._buffered_chunk = memoryview(b"data") + session._buffered_chunk_offset = 0 + + query_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "active", "X-Goog-Upload-Size-Received": "10"}, + body=b"", + ) + sess_transport = DummyAsyncSession([query_resp]) + + unseekable = mock.Mock() + unseekable.seekable.return_value = False + + with pytest.raises(UnseekableStreamError): + await session._recover(sess_transport, stream_obj=unseekable) + + assert session._buffered_chunk is None + + +@pytest.mark.asyncio +async def test_async_recover_stream_obj_none() -> None: + session = AsyncResumableUploadSession() + session._state._resumable_url = "https://upload.example.com/resumable-async" + session._buffered_chunk = None + + query_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "active", "X-Goog-Upload-Size-Received": "10"}, + body=b"", + ) + sess_transport = DummyAsyncSession([query_resp]) + + with pytest.raises(UnseekableStreamError, match="precedes active buffer"): + await session._recover(sess_transport, stream_obj=None) + + +@pytest.mark.asyncio +async def test_async_upload_already_finished_raises_value_error() -> None: + session = AsyncResumableUploadSession() + session._state._finished = True + session._state._resumable_url = "https://upload.example.com/resumable-async" + session.initiate = mock.AsyncMock() + + sess_transport = DummyAsyncSession([]) + with pytest.raises(ValueError, match="Upload completed without receiving a final response"): + await session.upload(stream=b"data", transport=sess_transport) + + +@pytest.mark.asyncio +async def test_async_resume_already_finished_raises_value_error() -> None: + session = AsyncResumableUploadSession() + session._state._finished = True + session._state._resumable_url = "https://upload.example.com/resumable-async" + session._recover = mock.AsyncMock() + + sess_transport = DummyAsyncSession([]) + op = session.resume( + upload_url="https://upload.example.com/resumable-async", + stream=b"data", + chunk_size=1024, + transport=sess_transport, + ) + with pytest.raises(ValueError, match="Upload resumed but completed without receiving a final response"): + await op + diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 4f5f9be5944b..2d312ee1bd9f 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -225,6 +225,15 @@ def test_protocol_state_chunk_missing_status_header(): state.process_chunk_response(200, {}, 10) +def test_protocol_state_chunk_cancelled_status(): + state = upload_state.ProtocolState( + resumable_url="https://upload.example.com/session" + ) + with pytest.raises(UploadCancelledError): + state.process_chunk_response(200, {"X-Goog-Upload-Status": "cancelled"}, 10) + assert state.invalid + + def test_protocol_state_query_and_cancel(): state = upload_state.ProtocolState( resumable_url="https://upload.example.com/session" @@ -238,6 +247,15 @@ def test_protocol_state_query_and_cancel(): assert received == 1024 assert state.bytes_uploaded == 1024 + # query with unknown status + state2 = upload_state.ProtocolState( + resumable_url="https://upload.example.com/session" + ) + received2 = state2.process_query_response( + 200, {"X-Goog-Upload-Status": "unknown"} + ) + assert received2 == 0 + method, url, headers, payload = state.build_cancel_request() assert headers["X-Goog-Upload-Command"] == "cancel" state.process_cancel_response(200, {}) @@ -1296,3 +1314,246 @@ def __bytes__(self) -> bytes: msg_instance = empty_pb2.Empty() res_msg = _format_response_payload(b"{}", response_type=msg_instance) assert isinstance(res_msg, empty_pb2.Empty) + + +class SlotException(Exception): + __slots__ = () + + +class NoTellStream: + def read(self, n): + return b"" + + +class TellErrorStream: + def read(self, n): + return b"" + def tell(self): + raise OSError("tell failed") + + +# CustomNoDictObj defines __slots__ as empty and has no parent class. +# It is used to test exception/metadata enrichment when the passed object +# does not have a __dict__ attribute (such as certain system exceptions). +class CustomNoDictObj: + __slots__ = () + + +class NoTellStream: + def read(self, n): + return b"" + + +class TellErrorStream: + def read(self, n): + return b"" + def tell(self): + raise OSError("tell failed") + + +class CustomReadStream: + def __init__(self, data): + self.data = data + def read(self, n): + return self.data + + +def test_sync_enrich_exception_no_dict(): + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=mock.sentinel.transport, + ) + exc = CustomNoDictObj() + session._enrich_exception(exc) + assert not hasattr(exc, "upload_url") + + +def test_sync_notify_progress_no_upload_url(): + session = ResumableUploadSession( + upload_url=None, + resumable_url=None, + transport=mock.sentinel.transport, + ) + session._notify_progress(common.ProgressState.STARTED) + + +def test_sync_should_retry_request_exception_not_retryable(): + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=mock.sentinel.transport, + ) + should_retry = session._get_retry_predicate() + exc = requests.exceptions.HTTPError("Non-retryable HTTP Error") + assert not should_retry(exc) + + +class NoSeekableButSeekStream: + def read(self, n): + return b"" + def seek(self, offset): + pass + + +class UnseekableReadStream: + def read(self, n): + return b"" + def seekable(self): + return False + + +class SeekableNoTellStream: + def read(self, n): + return b"" + def seekable(self): + return True + + +def test_sync_rewind_stream_no_seekable_attr(): + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=mock.sentinel.transport, + ) + stream = NoSeekableButSeekStream() + session._reposition_stream_offset(stream, 0) + + +def test_sync_recover_buffered_chunk_out_of_bounds(): + session = ResumableUploadSession() + session._buffered_chunk = memoryview(b"data") + session._buffered_chunk_offset = 0 + + unseekable = mock.Mock() + unseekable.seekable.return_value = False + + with pytest.raises(UnseekableStreamError): + session._reposition_stream_offset(unseekable, 10) + + assert session._buffered_chunk is None + + +def test_sync_prepare_stream_edge_cases(): + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + transport=mock.sentinel.transport, + ) + + # 1. NoTellStream: Has read, but no tell + stream1 = NoTellStream() + s_obj1, size1 = session._prepare_stream(stream1, size=None) + assert s_obj1 is stream1 + assert size1 is None + assert session._start_stream_offset == 0 + + # 2. TellErrorStream: Has read and tell, but tell raises OSError + stream2 = TellErrorStream() + s_obj2, size2 = session._prepare_stream(stream2, size=None) + assert s_obj2 is stream2 + assert size2 is None + assert session._start_stream_offset == 0 + + # 3. CustomReadStream: Has read, no getbuffer, no seekable + stream3 = CustomReadStream(b"hello") + s_obj3, size3 = session._prepare_stream(stream3, size=None) + assert s_obj3 is stream3 + assert size3 is None + + # 4. UnseekableReadStream: Has read, has seekable returning False + stream4 = UnseekableReadStream() + s_obj4, size4 = session._prepare_stream(stream4, size=None) + assert s_obj4 is stream4 + assert size4 is None + + # 5. SeekableNoTellStream: Has read, has seekable returning True, no tell + stream5 = SeekableNoTellStream() + s_obj5, size5 = session._prepare_stream(stream5, size=None) + assert s_obj5 is stream5 + assert size5 is None + + +def test_sync_transmit_all_chunks_captured_empty(): + session_transport = mock.create_autospec(requests.Session, instance=True) + chunk_resp = mock.create_autospec(requests.Response, instance=True) + chunk_resp.status_code = 200 + chunk_resp.content = b"{}" + chunk_resp.headers = {"X-Goog-Upload-Status": "final"} + session_transport.request.return_value = chunk_resp + + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + ) + session._state._resumable_url = "https://upload.example.com/resumable-123" + + session._captured_progress = None + stream_obj = io.BytesIO(b"data") + + # Consume the generator + list(session._transmit_all_chunks(session_transport, stream_obj, 4, [])) + + +def test_sync_upload_multiple_chunks(): + session_transport = mock.create_autospec(requests.Session, instance=True) + + # Start response + start_resp = mock.create_autospec(requests.Response, instance=True) + start_resp.status_code = 200 + start_resp.content = b"" + start_resp.headers = { + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + } + + # 1st chunk response: active + chunk1_resp = mock.create_autospec(requests.Response, instance=True) + chunk1_resp.status_code = 200 + chunk1_resp.content = b"" + chunk1_resp.headers = {"X-Goog-Upload-Status": "active"} + + # 2nd chunk response: final + chunk2_resp = mock.create_autospec(requests.Response, instance=True) + chunk2_resp.status_code = 200 + chunk2_resp.content = b"{}" + chunk2_resp.headers = {"X-Goog-Upload-Status": "final"} + + session_transport.request.side_effect = [start_resp, chunk1_resp, chunk2_resp] + + config = ResumableUploadConfig(chunk_size=5) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + ) + + res = session.upload(stream=b"0123456789", transport=session_transport) + assert res is chunk2_resp + assert session.bytes_uploaded == 10 + assert session_transport.request.call_count == 3 + + +def test_sync_format_response_payload_unsupported_type(): + from google.api_core.resumable_transfer.upload import _format_response_payload + res = _format_response_payload(b"some content", "unsupported") + assert res == b"some content" + + +def test_sync_prepare_stream_explicit_size(): + session = ResumableUploadSession(upload_url="https://api.example.com/init") + + # 1. bytes with explicit size + _, computed_size = session._prepare_stream(b"abcd", size=4) + assert computed_size == 4 + + # 2. Iterable with explicit size + _, computed_size = session._prepare_stream([b"ab", b"cd"], size=4) + assert computed_size == 4 + + # 3. BinaryIO with explicit size + _, computed_size = session._prepare_stream(io.BytesIO(b"abcd"), size=4) + assert computed_size == 4 + + +def test_state_process_chunk_response_unknown_status(): + from google.api_core.resumable_transfer.upload_state import ProtocolState + state = ProtocolState(upload_url="https://api.example.com/init") + state.process_chunk_response(200, {"X-Goog-Upload-Status": "unknown"}, 100) + assert state.bytes_uploaded == 0 + assert not state.finished + From fd49ae56a971bb0e48308bfa3b833f24e2e3d873 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Sep 2026 16:15:23 +0000 Subject: [PATCH 22/43] lint --- .../asyncio/test_resumable_transfer_async.py | 64 +++++++++++++++---- .../tests/unit/test_resumable_transfer.py | 13 ++-- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index b4d1b7709b1a..82fc7281691d 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -1022,6 +1022,7 @@ def test_async_resume_rejects_invalid_stream_types(invalid_stream: Any) -> None: stream=invalid_stream, ) + # CustomNoDictObj defines __slots__ as empty and has no parent class. # It is used to test exception/metadata enrichment when the passed object # does not have a __dict__ attribute (such as certain system exceptions). @@ -1278,7 +1279,9 @@ async def test_async_recover_stream_errors() -> None: @pytest.mark.asyncio async def test_async_upload_with_timeout_and_deadline() -> None: - future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=60) + future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + seconds=60 + ) config = ResumableUploadConfig(timeout=30.0, deadline=future_deadline) start_resp = DummyAsyncResponse( @@ -1289,7 +1292,9 @@ async def test_async_upload_with_timeout_and_deadline() -> None: }, body=b"", ) - resp = DummyAsyncResponse(status=200, headers={"X-Goog-Upload-Status": "final"}, body=b"{}") + resp = DummyAsyncResponse( + status=200, headers={"X-Goog-Upload-Status": "final"}, body=b"{}" + ) sess_transport = DummyAsyncSession([start_resp, resp]) session = AsyncResumableUploadSession( @@ -1307,7 +1312,9 @@ async def test_async_upload_with_timeout_and_deadline() -> None: @pytest.mark.asyncio async def test_async_transmit_chunk_timeout_errors() -> None: # 1. TimeoutError raises TransferStalledError when remaining is > 0 - future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=60) + future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + seconds=60 + ) config = ResumableUploadConfig(deadline=future_deadline) class TimeoutAsyncSession: @@ -1317,6 +1324,7 @@ def __init__(self): def request(self, *args, **kwargs): self.calls += 1 if self.calls == 1: + class StartContext: async def __aenter__(self): class Resp: @@ -1325,18 +1333,25 @@ class Resp: "X-Goog-Upload-Status": "active", "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", } + async def read(self): return b"" + return Resp() + async def __aexit__(self, exc_type, exc, tb): pass + return StartContext() else: + class TimeoutContext: async def __aenter__(self): raise asyncio.TimeoutError("timeout") + async def __aexit__(self, exc_type, exc, tb): pass + return TimeoutContext() session = AsyncResumableUploadSession( @@ -1350,7 +1365,9 @@ async def __aexit__(self, exc_type, exc, tb): await session.upload(stream=b"data") # 2. TimeoutError raises DeadlineExceeded when remaining <= 0 - past_deadline = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=10) + past_deadline = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + seconds=10 + ) config2 = ResumableUploadConfig(deadline=past_deadline) session2 = AsyncResumableUploadSession( @@ -1417,6 +1434,7 @@ def read(self, n): class LocalCustomReadStream: def __init__(self, data): self.data = data + def read(self, n): return self.data @@ -1471,7 +1489,9 @@ async def test_async_upload_no_stall_config() -> None: }, body=b"", ) - resp = DummyAsyncResponse(status=200, headers={"X-Goog-Upload-Status": "final"}, body=b"{}") + resp = DummyAsyncResponse( + status=200, headers={"X-Goog-Upload-Status": "final"}, body=b"{}" + ) sess_transport = DummyAsyncSession([start_resp, resp]) session = AsyncResumableUploadSession( @@ -1486,8 +1506,12 @@ async def test_async_upload_no_stall_config() -> None: @pytest.mark.asyncio async def test_async_transmit_chunk_timeout_errors_no_stall() -> None: # 1. TimeoutError raises TransferStalledError when remaining is > 0 and no stall control - future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=60) - config = ResumableUploadConfig(deadline=future_deadline, stall_minimum_rate=0, stall_timeout=0) + future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + seconds=60 + ) + config = ResumableUploadConfig( + deadline=future_deadline, stall_minimum_rate=0, stall_timeout=0 + ) class TimeoutAsyncSession: def __init__(self): @@ -1496,6 +1520,7 @@ def __init__(self): def request(self, *args, **kwargs): self.calls += 1 if self.calls == 1: + class StartContext: async def __aenter__(self): class Resp: @@ -1504,18 +1529,25 @@ class Resp: "X-Goog-Upload-Status": "active", "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", } + async def read(self): return b"" + return Resp() + async def __aexit__(self, exc_type, exc, tb): pass + return StartContext() else: + class TimeoutContext: async def __aenter__(self): raise asyncio.TimeoutError("timeout") + async def __aexit__(self, exc_type, exc, tb): pass + return TimeoutContext() session = AsyncResumableUploadSession( @@ -1529,8 +1561,12 @@ async def __aexit__(self, exc_type, exc, tb): await session.upload(stream=b"data") # 2. TimeoutError raises DeadlineExceeded when remaining <= 0 and no stall control - past_deadline = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=10) - config2 = ResumableUploadConfig(deadline=past_deadline, stall_minimum_rate=0, stall_timeout=0) + past_deadline = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( + seconds=10 + ) + config2 = ResumableUploadConfig( + deadline=past_deadline, stall_minimum_rate=0, stall_timeout=0 + ) session2 = AsyncResumableUploadSession( upload_url="https://api.example.com/start", @@ -1592,7 +1628,9 @@ async def test_async_upload_already_finished_raises_value_error() -> None: session.initiate = mock.AsyncMock() sess_transport = DummyAsyncSession([]) - with pytest.raises(ValueError, match="Upload completed without receiving a final response"): + with pytest.raises( + ValueError, match="Upload completed without receiving a final response" + ): await session.upload(stream=b"data", transport=sess_transport) @@ -1610,6 +1648,8 @@ async def test_async_resume_already_finished_raises_value_error() -> None: chunk_size=1024, transport=sess_transport, ) - with pytest.raises(ValueError, match="Upload resumed but completed without receiving a final response"): + with pytest.raises( + ValueError, + match="Upload resumed but completed without receiving a final response", + ): await op - diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 2d312ee1bd9f..11eac0129a26 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -251,9 +251,7 @@ def test_protocol_state_query_and_cancel(): state2 = upload_state.ProtocolState( resumable_url="https://upload.example.com/session" ) - received2 = state2.process_query_response( - 200, {"X-Goog-Upload-Status": "unknown"} - ) + received2 = state2.process_query_response(200, {"X-Goog-Upload-Status": "unknown"}) assert received2 == 0 method, url, headers, payload = state.build_cancel_request() @@ -1328,6 +1326,7 @@ def read(self, n): class TellErrorStream: def read(self, n): return b"" + def tell(self): raise OSError("tell failed") @@ -1347,6 +1346,7 @@ def read(self, n): class TellErrorStream: def read(self, n): return b"" + def tell(self): raise OSError("tell failed") @@ -1354,6 +1354,7 @@ def tell(self): class CustomReadStream: def __init__(self, data): self.data = data + def read(self, n): return self.data @@ -1390,6 +1391,7 @@ def test_sync_should_retry_request_exception_not_retryable(): class NoSeekableButSeekStream: def read(self, n): return b"" + def seek(self, offset): pass @@ -1397,6 +1399,7 @@ def seek(self, offset): class UnseekableReadStream: def read(self, n): return b"" + def seekable(self): return False @@ -1404,6 +1407,7 @@ def seekable(self): class SeekableNoTellStream: def read(self, n): return b"" + def seekable(self): return True @@ -1530,6 +1534,7 @@ def test_sync_upload_multiple_chunks(): def test_sync_format_response_payload_unsupported_type(): from google.api_core.resumable_transfer.upload import _format_response_payload + res = _format_response_payload(b"some content", "unsupported") assert res == b"some content" @@ -1552,8 +1557,8 @@ def test_sync_prepare_stream_explicit_size(): def test_state_process_chunk_response_unknown_status(): from google.api_core.resumable_transfer.upload_state import ProtocolState + state = ProtocolState(upload_url="https://api.example.com/init") state.process_chunk_response(200, {"X-Goog-Upload-Status": "unknown"}, 100) assert state.bytes_uploaded == 0 assert not state.finished - From 08f97ac23a23de731e4d9126be4eff2ec681e865 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Sep 2026 16:50:51 +0000 Subject: [PATCH 23/43] mypy --- .../api_core/resumable_transfer/upload.py | 5 +- .../resumable_transfer/upload_async.py | 5 +- .../asyncio/test_resumable_transfer_async.py | 107 ++++++++++-------- .../tests/unit/test_resumable_transfer.py | 26 +---- 4 files changed, 69 insertions(+), 74 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 39089d40eb17..8bfb8dd11388 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -195,9 +195,8 @@ def _enrich_exception(self, exc: BaseException) -> None: Args: exc: Exception instance to augment with upload_url and chunk_size. """ - if hasattr(exc, "__dict__"): - setattr(exc, "upload_url", self.upload_url) - setattr(exc, "chunk_size", self.chunk_size) + setattr(exc, "upload_url", self.upload_url) + setattr(exc, "chunk_size", self.chunk_size) def _notify_progress(self, state: common.ProgressState) -> None: """Notifies progress with current upload status. diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index cba5ad3e5ca9..c0b46616a339 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -208,9 +208,8 @@ def _enrich_exception(self, exc: BaseException) -> None: Args: exc: Exception instance to augment with upload_url and chunk_size. """ - if hasattr(exc, "__dict__"): - setattr(exc, "upload_url", self.upload_url) - setattr(exc, "chunk_size", self.chunk_size) + setattr(exc, "upload_url", self.upload_url) + setattr(exc, "chunk_size", self.chunk_size) def _notify_progress( self, state: common.ProgressState, queue: Optional[asyncio.Queue] = None diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 82fc7281691d..3cc224f7e04e 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -1023,20 +1023,14 @@ def test_async_resume_rejects_invalid_stream_types(invalid_stream: Any) -> None: ) -# CustomNoDictObj defines __slots__ as empty and has no parent class. -# It is used to test exception/metadata enrichment when the passed object -# does not have a __dict__ attribute (such as certain system exceptions). -class CustomNoDictObj: - __slots__ = () - - -def test_async_enrich_exception_without_dict() -> None: - """Verifies that _enrich_exception handles objects without __dict__.""" +def test_async_enrich_exception() -> None: + """Verifies that _enrich_exception attaches upload_url and chunk_size.""" session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", ) - exc = CustomNoDictObj() + exc = RuntimeError("test error") session._enrich_exception(exc) + assert getattr(exc, "upload_url") == "https://api.example.com/start" def test_async_notify_progress_branches() -> None: @@ -1146,7 +1140,10 @@ async def fail_runtime(): assert attempts_runtime == 2 # max_attempts=0 covers loop bypass falling through - res = await session._async_retry(lambda: 123, max_attempts=0) + async def dummy_async_func() -> int: + return 123 + + res = await session._async_retry(dummy_async_func, max_attempts=0) assert res is None @@ -1376,10 +1373,10 @@ async def __aexit__(self, exc_type, exc, tb): transport=TimeoutAsyncSession(), ) session2._state._resumable_url = "https://upload.example.com/resumable-async" - session2._get_deadline_remaining = lambda: -5.0 - with pytest.raises(exceptions.DeadlineExceeded): - await session2.upload(stream=b"data") + with mock.patch.object(session2, "_get_deadline_remaining", return_value=-5.0): + with pytest.raises(exceptions.DeadlineExceeded): + await session2.upload(stream=b"data") @pytest.mark.asyncio @@ -1427,27 +1424,47 @@ async def test_async_prepare_async_reader_additional_branches() -> None: upload_url="https://api.example.com/start", ) - class LocalNoTellStream: - def read(self, n): - return b"" + # Inherit from concrete io.BytesIO so mypy recognizes these test streams as + # valid BinaryIO instances without requiring cast() or type: ignore (since + # typing.BinaryIO is an abstract class). + class LocalNoTellStream(io.BytesIO): + """Simulates a stream that has read() but lacks getbuffer() and tell(). + + Overrides __getattribute__ to raise AttributeError for "tell" and + "getbuffer" so that hasattr(stream, "tell") and hasattr(stream, "getbuffer") + evaluate to False at runtime while remaining mypy-compliant. + """ - class LocalCustomReadStream: - def __init__(self, data): - self.data = data + def __getattribute__(self, name: str) -> Any: + if name in ("tell", "getbuffer"): + raise AttributeError(f"no {name}") + return super().__getattribute__(name) - def read(self, n): - return self.data + class LocalCustomReadStream(io.BytesIO): + """Simulates a stream that lacks getbuffer() and where tell() raises OSError. + + Used to verify that _prepare_async_reader gracefully catches OSError + when attempting to record the starting stream offset via tell(). + """ + + def __getattribute__(self, name: str) -> Any: + if name == "getbuffer": + raise AttributeError("no getbuffer") + return super().__getattribute__(name) + + def tell(self) -> int: + raise OSError("tell failed") # 1. bytes stream with explicit size reader_fn1, size1, obj1 = session._prepare_async_reader(b"data", size=4) assert size1 == 4 # 2. NoTellStream: read but no tell - stream2 = LocalNoTellStream() + stream2 = LocalNoTellStream(b"") reader_fn2, size2, obj2 = session._prepare_async_reader(stream2, size=None) assert size2 is None - # 3. CustomReadStream: read, but no getbuffer and no tell + # 3. CustomReadStream: read, but no getbuffer and tell raises OSError stream3 = LocalCustomReadStream(b"hello") reader_fn3, size3, obj3 = session._prepare_async_reader(stream3, size=None) assert size3 is None @@ -1574,10 +1591,10 @@ async def __aexit__(self, exc_type, exc, tb): transport=TimeoutAsyncSession(), ) session2._state._resumable_url = "https://upload.example.com/resumable-async" - session2._get_deadline_remaining = lambda: -5.0 - with pytest.raises(exceptions.DeadlineExceeded): - await session2.upload(stream=b"data") + with mock.patch.object(session2, "_get_deadline_remaining", return_value=-5.0): + with pytest.raises(exceptions.DeadlineExceeded): + await session2.upload(stream=b"data") @pytest.mark.asyncio @@ -1625,13 +1642,13 @@ async def test_async_upload_already_finished_raises_value_error() -> None: session = AsyncResumableUploadSession() session._state._finished = True session._state._resumable_url = "https://upload.example.com/resumable-async" - session.initiate = mock.AsyncMock() - sess_transport = DummyAsyncSession([]) - with pytest.raises( - ValueError, match="Upload completed without receiving a final response" - ): - await session.upload(stream=b"data", transport=sess_transport) + with mock.patch.object(session, "initiate", new_callable=mock.AsyncMock): + sess_transport = DummyAsyncSession([]) + with pytest.raises( + ValueError, match="Upload completed without receiving a final response" + ): + await session.upload(stream=b"data", transport=sess_transport) @pytest.mark.asyncio @@ -1639,17 +1656,17 @@ async def test_async_resume_already_finished_raises_value_error() -> None: session = AsyncResumableUploadSession() session._state._finished = True session._state._resumable_url = "https://upload.example.com/resumable-async" - session._recover = mock.AsyncMock() sess_transport = DummyAsyncSession([]) - op = session.resume( - upload_url="https://upload.example.com/resumable-async", - stream=b"data", - chunk_size=1024, - transport=sess_transport, - ) - with pytest.raises( - ValueError, - match="Upload resumed but completed without receiving a final response", - ): - await op + with mock.patch.object(session, "_recover", new_callable=mock.AsyncMock): + op = session.resume( + upload_url="https://upload.example.com/resumable-async", + stream=b"data", + chunk_size=1024, + transport=sess_transport, + ) + with pytest.raises( + ValueError, + match="Upload resumed but completed without receiving a final response", + ): + await op diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 11eac0129a26..8a79cb1094c7 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -1331,26 +1331,6 @@ def tell(self): raise OSError("tell failed") -# CustomNoDictObj defines __slots__ as empty and has no parent class. -# It is used to test exception/metadata enrichment when the passed object -# does not have a __dict__ attribute (such as certain system exceptions). -class CustomNoDictObj: - __slots__ = () - - -class NoTellStream: - def read(self, n): - return b"" - - -class TellErrorStream: - def read(self, n): - return b"" - - def tell(self): - raise OSError("tell failed") - - class CustomReadStream: def __init__(self, data): self.data = data @@ -1359,14 +1339,14 @@ def read(self, n): return self.data -def test_sync_enrich_exception_no_dict(): +def test_sync_enrich_exception(): session = ResumableUploadSession( upload_url="https://api.example.com/init", transport=mock.sentinel.transport, ) - exc = CustomNoDictObj() + exc = RuntimeError("test error") session._enrich_exception(exc) - assert not hasattr(exc, "upload_url") + assert getattr(exc, "upload_url") == "https://api.example.com/init" def test_sync_notify_progress_no_upload_url(): From 09aff46c2366c320b30df1fedb0b14259cb6f514 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Sep 2026 17:27:21 +0000 Subject: [PATCH 24/43] add regression test for missing data --- .../api_core/resumable_transfer/upload.py | 3 +- .../resumable_transfer/upload_async.py | 3 +- .../asyncio/test_resumable_transfer_async.py | 99 ++++++++++++++++++- .../tests/unit/test_resumable_transfer.py | 92 ++++++++++++++++- 4 files changed, 193 insertions(+), 4 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 8bfb8dd11388..cca54ca82494 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -562,7 +562,8 @@ def do_http() -> requests.Response: is_recoverable = ( isinstance(exc, exceptions.GoogleAPICallError) - and exc.code in common.RECOVERABLE_STATUS_CODES + and exc.code + in (common.RECOVERABLE_STATUS_CODES + common.RETRYABLE_STATUS_CODES) ) or isinstance(exc, exceptions.MissingStatusHeaderError) if is_recoverable: diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index c0b46616a339..b0e9de43756b 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -520,7 +520,8 @@ async def do_http(): is_recoverable = ( isinstance(exc, exceptions.GoogleAPICallError) - and exc.code in common.RECOVERABLE_STATUS_CODES + and exc.code + in (common.RECOVERABLE_STATUS_CODES + common.RETRYABLE_STATUS_CODES) ) or isinstance(exc, exceptions.MissingStatusHeaderError) if is_recoverable: diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 3cc224f7e04e..e372bf9f0cde 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -1027,10 +1027,11 @@ def test_async_enrich_exception() -> None: """Verifies that _enrich_exception attaches upload_url and chunk_size.""" session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", + resumable_url="https://upload.example.com/resumable-async", ) exc = RuntimeError("test error") session._enrich_exception(exc) - assert getattr(exc, "upload_url") == "https://api.example.com/start" + assert getattr(exc, "upload_url") == "https://upload.example.com/resumable-async" def test_async_notify_progress_branches() -> None: @@ -1670,3 +1671,99 @@ async def test_async_resume_already_finished_raises_value_error() -> None: match="Upload resumed but completed without receiving a final response", ): await op + + +@pytest.mark.asyncio +async def test_async_partial_chunk_recovery_does_not_prematurely_finalize() -> None: + """Ensure that retrying a partially committed chunk (len < chunk_size) does not prematurely finalize. + + When a 4-byte chunk (b"0123") partially succeeds (server commits 2 bytes) + and is retried, ensure that the remaining 2 bytes (b"23") are sent with + "upload" rather than "upload, finalize" so the remaining payload (b"45") + is not dropped. + """ + server_received_bytes = bytearray() + + class StatefulAsyncTransport: + def __init__(self) -> None: + self.call_count = 0 + + def request( + self, + method: str, + url: str, + data: Any = None, + headers: Optional[Mapping[str, str]] = None, + **kwargs: Any, + ) -> DummyAsyncResponse: + self.call_count += 1 + cmd = headers.get("X-Goog-Upload-Command", "") if headers else "" + + # Request 1: start session + if self.call_count == 1: + assert cmd == "start" + return DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + + # Request 2: initial Chunk 1 (b"0123") -> server commits partial 2 bytes (b"01"), then fails 503 + if self.call_count == 2: + assert cmd == "upload" + assert bytes(data) == b"0123" + server_received_bytes.extend(b"01") + return DummyAsyncResponse( + status=503, + headers={}, + body=b"Service Unavailable", + ) + + # Request 3: recovery query -> server reports 2 committed bytes + if self.call_count == 3: + assert cmd == "query" + return DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": str(len(server_received_bytes)), + }, + body=b"", + ) + + # Subsequent upload requests (Request 4: remaining b"23", Request 5: final b"45") + if data: + server_received_bytes.extend(bytes(data)) + + if "finalize" in cmd: + return DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b"{}", + ) + return DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "active"}, + body=b"", + ) + + transport = StatefulAsyncTransport() + config = ResumableUploadConfig(chunk_size=4) + session = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=transport, + ) + + # Disable HTTP-level _async_retry so 503 immediately triggers protocol-level _recover() + async def no_retry(coro_fn: Any, max_attempts: int = 1) -> Any: + return await coro_fn() + + with mock.patch.object(session, "_async_retry", side_effect=no_retry): + await session.upload(stream=b"012345") + + # Verify no data loss occurred: server must receive all 6 bytes (b"012345"), not truncated b"0123" + assert bytes(server_received_bytes) == b"012345" diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 8a79cb1094c7..552909179667 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -1342,11 +1342,12 @@ def read(self, n): def test_sync_enrich_exception(): session = ResumableUploadSession( upload_url="https://api.example.com/init", + resumable_url="https://upload.example.com/resumable-123", transport=mock.sentinel.transport, ) exc = RuntimeError("test error") session._enrich_exception(exc) - assert getattr(exc, "upload_url") == "https://api.example.com/init" + assert getattr(exc, "upload_url") == "https://upload.example.com/resumable-123" def test_sync_notify_progress_no_upload_url(): @@ -1542,3 +1543,92 @@ def test_state_process_chunk_response_unknown_status(): state.process_chunk_response(200, {"X-Goog-Upload-Status": "unknown"}, 100) assert state.bytes_uploaded == 0 assert not state.finished + + +def test_sync_partial_chunk_recovery_does_not_prematurely_finalize(): + """Ensure that retrying a partially committed chunk (len < chunk_size) does not prematurely finalize. + + When a 4-byte chunk (b"0123") partially succeeds (server commits 2 bytes) + and is retried, ensure that the remaining 2 bytes (b"23") are sent with + "upload" rather than "upload, finalize" so the remaining payload (b"45") + is not dropped. + """ + server_received_bytes = bytearray() + call_count = 0 + + def handle_request(method, url, data=None, headers=None, **kwargs): + nonlocal call_count + call_count += 1 + cmd = headers.get("X-Goog-Upload-Command", "") if headers else "" + + resp = mock.create_autospec(requests.Response, instance=True) + resp.request = mock.Mock(method=method, url=url) + resp.content = b"" + + # Request 1: start session + if call_count == 1: + assert cmd == "start" + resp.status_code = 200 + resp.ok = True + resp.headers = { + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + } + return resp + + # Request 2: initial Chunk 1 (b"0123") -> server commits partial 2 bytes (b"01"), then fails 503 + if call_count == 2: + assert cmd == "upload" + assert bytes(data) == b"0123" + server_received_bytes.extend(b"01") + resp.status_code = 503 + resp.ok = False + resp.headers = {} + resp.json.return_value = { + "error": {"code": 503, "message": "Service Unavailable"} + } + return resp + + # Request 3: recovery query -> server reports 2 committed bytes + if call_count == 3: + assert cmd == "query" + resp.status_code = 200 + resp.ok = True + resp.headers = { + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": str(len(server_received_bytes)), + } + return resp + + # Subsequent upload requests (Request 4: remaining b"23", Request 5: final b"45") + if data: + server_received_bytes.extend(bytes(data)) + + resp.status_code = 200 + resp.ok = True + if "finalize" in cmd: + resp.headers = {"X-Goog-Upload-Status": "final"} + resp.content = b"{}" + else: + resp.headers = {"X-Goog-Upload-Status": "active"} + return resp + + session_transport = mock.create_autospec(requests.Session, instance=True) + session_transport.request.side_effect = handle_request + + # Resumable uploads have two layers of retry: + # 1. HTTP-level retry (do_http): blindly re-sends the HTTP request on transient 503 errors. + # 2. Protocol-level recovery (_recover): triggered when HTTP retries exhaust; sends a "query" + # command to discover committed server offset and slices the active chunk buffer. + # Disable HTTP-level retry here so the 503 immediately triggers protocol-level _recover(). + retry_cfg = google.api_core.retry.Retry(predicate=lambda exc: False) + config = ResumableUploadConfig(chunk_size=4, retry=retry_cfg) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + ) + + session.upload(stream=b"012345", transport=session_transport) + + # Verify no data loss occurred: server must receive all 6 bytes (b"012345"), not truncated b"0123" + assert bytes(server_received_bytes) == b"012345" From a51f058bd3e915d8c0d6d58205f74579e719ac56 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Sep 2026 17:29:07 +0000 Subject: [PATCH 25/43] add comments --- .../tests/asyncio/test_resumable_transfer_async.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index e372bf9f0cde..dd7b74465eef 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -1758,7 +1758,11 @@ def request( transport=transport, ) - # Disable HTTP-level _async_retry so 503 immediately triggers protocol-level _recover() + # Resumable uploads have two layers of retry: + # 1. HTTP-level retry (do_http): blindly re-sends the HTTP request on transient 503 errors. + # 2. Protocol-level recovery (_recover): triggered when HTTP retries exhaust; sends a "query" + # command to discover committed server offset and slices the active chunk buffer. + # Disable HTTP-level _async_retry here so the 503 immediately triggers protocol-level _recover(). async def no_retry(coro_fn: Any, max_attempts: int = 1) -> Any: return await coro_fn() From f783a6fc34f8af82f651f29b1b5a7de892d2eef5 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Sep 2026 20:27:02 +0000 Subject: [PATCH 26/43] resolve issue where data may be lost on partial commit recovery --- .../api_core/resumable_transfer/upload.py | 21 ++++++++++++++----- .../resumable_transfer/upload_async.py | 21 ++++++++++++++----- packages/google-api-core/noxfile.py | 2 +- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index cca54ca82494..19b11bf3d228 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -140,6 +140,7 @@ def __init__( # In-memory zero-copy buffer (never discard chunk until confirmed) self._buffered_chunk: Optional[memoryview] = None self._buffered_chunk_offset: int = 0 + self._buffered_chunk_is_last: bool = False self._start_stream_offset: int = 0 # Stall control tracking via monotonic clock @@ -480,20 +481,30 @@ def _transmit_chunk( def do_transmit() -> requests.Response: chunk_size = self._state.chunk_size - # Retain active chunk in zero-copy buffer if not present + # Retain active chunk in zero-copy buffer if not present. + # Ensure that EOF status (_buffered_chunk_is_last) is computed once + # when reading from the stream and preserved across _recover() retries. + # On partial server commit, _recover() slices _buffered_chunk in-place + # to the uncommitted tail. Preserving _buffered_chunk_is_last ensures + # that a sliced tail smaller than chunk_size is not prematurely + # treated as the final chunk when unread bytes remain in the stream. if self._buffered_chunk is None: raw_bytes = stream.read(chunk_size) if not raw_bytes: raw_bytes = b"" self._buffered_chunk = memoryview(raw_bytes) self._buffered_chunk_offset = self._state.bytes_uploaded + is_eof = len(raw_bytes) < chunk_size + if ( + size is not None + and self._state.bytes_uploaded + len(raw_bytes) >= size + ): + is_eof = True + self._buffered_chunk_is_last = is_eof data = self._buffered_chunk data_len = len(data) - - is_last = data_len < chunk_size - if size is not None and self._state.bytes_uploaded + data_len >= size: - is_last = True + is_last = self._buffered_chunk_is_last method, url, headers, payload = self._state.build_chunk_request( data=data, diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index b0e9de43756b..3b122b38b47d 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -159,6 +159,7 @@ def __init__( # In-memory zero-copy buffer self._buffered_chunk: Optional[memoryview] = None self._buffered_chunk_offset: int = 0 + self._buffered_chunk_is_last: bool = False self._start_stream_offset: int = 0 # Stall control tracking via monotonic clock @@ -391,20 +392,30 @@ async def _transmit_chunk( async def do_transmit(): chunk_size = self._state.chunk_size - # Retain active chunk in zero-copy buffer if not present + # Retain active chunk in zero-copy buffer if not present. + # Ensure that EOF status (_buffered_chunk_is_last) is computed once + # when reading from the stream and preserved across _recover() retries. + # On partial server commit, _recover() slices _buffered_chunk in-place + # to the uncommitted tail. Preserving _buffered_chunk_is_last ensures + # that a sliced tail smaller than chunk_size is not prematurely + # treated as the final chunk when unread bytes remain in the stream. if self._buffered_chunk is None: raw_bytes = await reader_fn(chunk_size) if not raw_bytes: raw_bytes = b"" self._buffered_chunk = memoryview(raw_bytes) self._buffered_chunk_offset = self._state.bytes_uploaded + is_eof = len(raw_bytes) < chunk_size + if ( + size is not None + and self._state.bytes_uploaded + len(raw_bytes) >= size + ): + is_eof = True + self._buffered_chunk_is_last = is_eof data = self._buffered_chunk data_len = len(data) - - is_last = data_len < chunk_size - if size is not None and self._state.bytes_uploaded + data_len >= size: - is_last = True + is_last = self._buffered_chunk_is_last method, url, headers, payload = self._state.build_chunk_request( data=data, diff --git a/packages/google-api-core/noxfile.py b/packages/google-api-core/noxfile.py index 4ae56dc0f008..dd15dff625eb 100644 --- a/packages/google-api-core/noxfile.py +++ b/packages/google-api-core/noxfile.py @@ -296,7 +296,7 @@ def default( "--cov=tests.unit", "--cov-append", "--cov-config=.coveragerc", - "--cov-report=", + "--cov-report=term", "--cov-fail-under=0", # Running individual tests with parallelism enabled is usually not helpful. "-n=auto", From dd4167ad7e961afa02a33883bfc6ac6fe280a60c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Sep 2026 21:17:19 +0000 Subject: [PATCH 27/43] address review feedback --- .../api_core/resumable_transfer/upload.py | 41 +++++++------------ .../tests/unit/test_resumable_transfer.py | 14 ++++--- 2 files changed, 22 insertions(+), 33 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 19b11bf3d228..ebc243b3c038 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -219,18 +219,11 @@ def _notify_progress(self, state: common.ProgressState) -> None: self._config.on_progress(progress) @contextlib.contextmanager - def _capture_progress( - self, - ) -> Generator[List[common.UploadProgress], None, None]: - """Intercepts progress events to buffer snapshots for generator consumers. - - Yields: - List buffering UploadProgress snapshots during generator execution. - """ - captured: List[common.UploadProgress] = [] - self._captured_progress = captured + def _capture_progress(self) -> Generator[None, None, None]: + """Initializes self._captured_progress to buffer snapshots for generator consumers.""" + self._captured_progress = [] try: - yield captured + yield finally: self._captured_progress = None @@ -645,7 +638,6 @@ def _transmit_all_chunks( transport: requests.Session, stream_obj: BinaryIO, computed_size: Optional[int], - captured: Optional[List[common.UploadProgress]] = None, ) -> Generator[common.UploadProgress, None, None]: """Transmits chunks until transfer completes, yielding buffered progress updates. @@ -653,7 +645,6 @@ def _transmit_all_chunks( transport: The requests session. stream_obj: Binary stream yielding upload chunks. computed_size: Total payload size in bytes if known. - captured: Optional buffer accumulating progress snapshots. Yields: UploadProgress snapshots for each transmission milestone. @@ -661,16 +652,16 @@ def _transmit_all_chunks( Raises: ValueError: If upload concludes without a server response. """ - if captured: - while captured: - yield captured.pop(0) + if self._captured_progress: + while self._captured_progress: + yield self._captured_progress.pop(0) final_resp = None while not self._state.finished and not self._state.invalid: final_resp = self._transmit_chunk(transport, stream_obj, computed_size) - if captured: - while captured: - yield captured.pop(0) + if self._captured_progress: + while self._captured_progress: + yield self._captured_progress.pop(0) if final_resp is None: raise ValueError("Upload completed without receiving a final response.") @@ -728,15 +719,13 @@ def iter_upload( GoogleAPICallError: If an unrecoverable API error occurs. """ sess = self._get_transport(transport) - with self._capture_progress() as captured: + with self._capture_progress(): try: stream_obj, computed_size = self._prepare_stream(stream, size) self.initiate( transport=sess, request_body=request_body, size=computed_size ) - yield from self._transmit_all_chunks( - sess, stream_obj, computed_size, captured - ) + yield from self._transmit_all_chunks(sess, stream_obj, computed_size) except Exception as exc: self._enrich_exception(exc) raise @@ -810,13 +799,11 @@ def iter_resume( self._state._chunk_size = chunk_size self._state._resumable_url = actual_url - with self._capture_progress() as captured: + with self._capture_progress(): try: stream_obj, computed_size = self._prepare_stream(stream, size) self._recover(sess, stream_obj) - yield from self._transmit_all_chunks( - sess, stream_obj, computed_size, captured - ) + yield from self._transmit_all_chunks(sess, stream_obj, computed_size) except Exception as exc: self._enrich_exception(exc) raise diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 552909179667..a5fa834d0a14 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -1024,12 +1024,14 @@ def test_sync_on_progress_and_capture(): config=config, ) session._state._resumable_url = "https://api.example.com/init" - with session._capture_progress() as captured: + with session._capture_progress(): session._notify_progress(common.ProgressState.UPLOADING) - assert len(captured) == 1 - assert captured[0].state == common.ProgressState.UPLOADING - assert callback_mock.called - assert callback_mock.call_args[0][0] is captured[0] + assert session._captured_progress is not None + assert len(session._captured_progress) == 1 + assert session._captured_progress[0].state == common.ProgressState.UPLOADING + assert callback_mock.called + assert callback_mock.call_args[0][0] is session._captured_progress[0] + assert session._captured_progress is None def test_sync_naive_deadline_tz(): @@ -1472,7 +1474,7 @@ def test_sync_transmit_all_chunks_captured_empty(): stream_obj = io.BytesIO(b"data") # Consume the generator - list(session._transmit_all_chunks(session_transport, stream_obj, 4, [])) + list(session._transmit_all_chunks(session_transport, stream_obj, 4)) def test_sync_upload_multiple_chunks(): From db1b9805e2b1090ac4a1d350e7bab32e306f4452 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Sep 2026 21:21:39 +0000 Subject: [PATCH 28/43] cover --- packages/google-api-core/noxfile.py | 2 +- .../tests/unit/test_resumable_transfer.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/google-api-core/noxfile.py b/packages/google-api-core/noxfile.py index dd15dff625eb..4ae56dc0f008 100644 --- a/packages/google-api-core/noxfile.py +++ b/packages/google-api-core/noxfile.py @@ -296,7 +296,7 @@ def default( "--cov=tests.unit", "--cov-append", "--cov-config=.coveragerc", - "--cov-report=term", + "--cov-report=", "--cov-fail-under=0", # Running individual tests with parallelism enabled is usually not helpful. "-n=auto", diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index a5fa834d0a14..492679799eae 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -1127,6 +1127,17 @@ def test_sync_update_stall_control_disabled(): session._update_stall_control(512, time.monotonic(), 5.0) assert session._aggregate_lag == 0.0 + # Ensure that _stall_timeout_started resets to None when transfer rate exceeds minimum rate (no lag). + active_config = ResumableUploadConfig(stall_minimum_rate=1024, stall_timeout=10.0) + active_session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=active_config, + ) + active_session._stall_timeout_started = 100.0 + active_session._update_stall_control(1024, time.monotonic(), 0.1) + assert active_session._aggregate_lag == 0.0 + assert active_session._stall_timeout_started is None + def test_sync_initiate_failure(): transport = mock.create_autospec(requests.Session, instance=True) From 8fa74b36b8e0de933145a7f93295fd86bc3fa429 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 15 Sep 2026 21:30:49 +0000 Subject: [PATCH 29/43] address review feedback --- .../api_core/resumable_transfer/upload.py | 43 +++++++++++++++++-- .../tests/unit/test_resumable_transfer.py | 11 ++++- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index ebc243b3c038..098795c80a17 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -55,6 +55,45 @@ class _RecoveryRetransmit(Exception): pass +class _IterableReader(io.BytesIO): + """Wraps an Iterable[bytes] as a non-seekable binary stream. + + Ensure that chunks are pulled lazily from the underlying iterator + on each read() call rather than buffering the entire iterable into memory. + Inherits from io.BytesIO so static type checkers recognize instances as + BinaryIO without casts. + """ + + def __init__(self, iterable: Iterable[bytes]) -> None: + super().__init__() + self._iterator = iter(iterable) + self._buffer = bytearray() + + def seekable(self) -> bool: + return False + + def tell(self) -> int: + raise OSError("Stream is not seekable") + + def read(self, size: Optional[int] = -1) -> bytes: + if size is None or size < 0: + for chunk in self._iterator: + self._buffer.extend(chunk) + result = bytes(self._buffer) + self._buffer.clear() + return result + + while len(self._buffer) < size: + try: + chunk = next(self._iterator) + self._buffer.extend(chunk) + except StopIteration: + break + result = bytes(self._buffer[:size]) + del self._buffer[:size] + return result + + @dataclasses.dataclass class ResumableUploadConfig: """Configuration options for a resumable upload. @@ -828,9 +867,7 @@ def _prepare_stream( if computed_size is None: computed_size = len(stream) elif not hasattr(stream, "read") and isinstance(stream, Iterable): - stream_obj = io.BytesIO(b"".join(stream)) - if computed_size is None: - computed_size = stream_obj.getbuffer().nbytes + stream_obj = _IterableReader(stream) elif hasattr(stream, "read"): stream_obj = cast(BinaryIO, stream) if computed_size is None: diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 492679799eae..f6262df3243d 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -918,8 +918,15 @@ def test_sync_prepare_stream_seekable_and_iterable(): ) stream_obj, computed_size = session._prepare_stream([b"hello ", b"world"], None) - assert stream_obj.read() == b"hello world" - assert computed_size == 11 + assert stream_obj.seekable() is False + assert stream_obj.read(4) == b"hell" + assert stream_obj.read(4) == b"o wo" + assert stream_obj.read(4) == b"rld" + assert stream_obj.read(4) == b"" + assert computed_size is None + + stream_obj_all, _ = session._prepare_stream([b"hello ", b"world"], None) + assert stream_obj_all.read() == b"hello world" class CustomSeekable: def __init__(self, data: bytes): From ae8a644e662cc3237dd5670fbf4cfe398fd75bf0 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 14:11:33 +0000 Subject: [PATCH 30/43] Address review feedback --- .../api_core/resumable_transfer/__init__.py | 6 +- .../api_core/resumable_transfer/common.py | 128 +++- .../api_core/resumable_transfer/upload.py | 612 +++++++++--------- .../resumable_transfer/upload_async.py | 590 ++++++++++------- .../asyncio/test_resumable_transfer_async.py | 134 ++-- .../tests/unit/test_resumable_transfer.py | 131 +++- 6 files changed, 942 insertions(+), 659 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/__init__.py b/packages/google-api-core/google/api_core/resumable_transfer/__init__.py index 041ef2a6cf72..3b43e19d2052 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/__init__.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/__init__.py @@ -25,13 +25,11 @@ DEFAULT_CHUNK_SIZE, Command, ProgressState, + ResumableUploadConfig, Status, UploadProgress, ) -from google.api_core.resumable_transfer.upload import ( - ResumableUploadConfig, - ResumableUploadSession, -) +from google.api_core.resumable_transfer.upload import ResumableUploadSession from google.api_core.resumable_transfer.upload_async import ( AsyncResumableUploadSession, AsyncUploadOperation, diff --git a/packages/google-api-core/google/api_core/resumable_transfer/common.py b/packages/google-api-core/google/api_core/resumable_transfer/common.py index 0859cde83de3..f426f5db26e8 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/common.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/common.py @@ -12,11 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Common constants and headers for Resumable Upload protocol.""" +"""Common constants, headers, and shared configuration for Resumable Upload protocol.""" import dataclasses +import datetime import enum -from typing import Optional +from typing import Any, Callable, Mapping, Optional, Sequence, Tuple, Union + +import google.protobuf.message +import proto +from google.protobuf import json_format + +import google.api_core.retry # Default chunk size: 10 MiB DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024 @@ -87,3 +94,120 @@ class UploadProgress: # HTTP status codes indicating state consistency errors requiring recovery RECOVERABLE_STATUS_CODES = (400, 409, 412, 416) + + +@dataclasses.dataclass +class ResumableUploadConfig: + """Configuration options for a resumable upload. + + Attributes: + chunk_size: Size in bytes for each uploaded data chunk. Defaults to 10 MiB. + start_timeout: Local per-request timeout in seconds for start request. + start_retry: Custom retry policy for the start request. + stall_minimum_rate: Minimum transfer rate in bytes per second. Defaults to 64 KiB/s. + stall_timeout: Stall duration threshold in seconds. Defaults to 120s. + headers: Additional HTTP headers dispatched exclusively with start request. + deadline: Overall global deadline for the upload process. + timeout: Fallback per-request timeout. + retry: Fallback retry policy. + on_progress: Callback function receiving UploadProgress notifications. + response_type: Optional message class (proto.Message or google.protobuf.message.Message), + callable deserializer, or None to return raw response. + content_type: MIME type of the stream payload. + """ + + chunk_size: int = DEFAULT_CHUNK_SIZE + start_timeout: Optional[float] = None + start_retry: Optional[ + Union[google.api_core.retry.Retry, google.api_core.retry.AsyncRetry] + ] = None + stall_minimum_rate: int = 64 * 1024 + stall_timeout: float = 120.0 + headers: Optional[Union[Mapping[str, str], Sequence[Tuple[str, str]]]] = None + deadline: Optional[datetime.datetime] = None + timeout: Optional[float] = None + retry: Optional[ + Union[ + google.api_core.retry.Retry, + google.api_core.retry.StreamingRetry, + google.api_core.retry.AsyncRetry, + google.api_core.retry.AsyncStreamingRetry, + ] + ] = None + on_progress: Optional[Callable[[UploadProgress], None]] = None + response_type: Optional[Any] = None + content_type: Optional[str] = None + + def __post_init__(self) -> None: + """Normalizes fallback timeouts and retry policies.""" + if self.start_timeout is not None and self.timeout is None: + self.timeout = self.start_timeout + elif self.timeout is not None and self.start_timeout is None: + self.start_timeout = self.timeout + + if self.start_retry is not None and self.retry is None: + self.retry = self.start_retry + elif ( + self.start_retry is None + and self.retry is not None + and not isinstance( + self.retry, + ( + google.api_core.retry.StreamingRetry, + google.api_core.retry.AsyncStreamingRetry, + ), + ) + ): + self.start_retry = self.retry + + @property + def start_headers(self) -> Optional[Sequence[Tuple[str, str]]]: + """Returns normalized additional headers for the start request.""" + if self.headers is None: + return None + if isinstance(self.headers, Mapping): + return list(self.headers.items()) + return list(self.headers) + + +def _format_response_payload( + response: Union[Any, bytes], + response_type: Optional[Any], +) -> Any: + """Formats raw response or bytes into protobuf or proto-plus message type if configured. + + Args: + response: Raw HTTP response object or response body bytes. + response_type: Deserializer callable, proto.Message class, or + google.protobuf.message.Message class or instance. + + Returns: + Deserialized protobuf message or the raw response object / bytes. + """ + if response_type is None: + return response + + content: bytes + if isinstance(response, bytes): + content = response + elif hasattr(response, "content"): + content = response.content + else: + content = bytes(response) + + from_json_fn = getattr(response_type, "from_json", None) + if callable(from_json_fn): + if isinstance(response_type, type) and issubclass(response_type, proto.Message): + return from_json_fn(content, ignore_unknown_fields=True) + return from_json_fn(content) + if isinstance(response_type, type) and issubclass( + response_type, google.protobuf.message.Message + ): + instance = response_type() + return json_format.Parse(content, instance, ignore_unknown_fields=True) + if isinstance(response_type, google.protobuf.message.Message): + return json_format.Parse(content, response_type, ignore_unknown_fields=True) + if callable(response_type): + return response_type(content) + + return response diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 098795c80a17..af9cfb7eae6f 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -14,8 +14,6 @@ """Synchronous Resumable Upload session and helpers using requests.""" -import contextlib -import dataclasses import datetime import io import logging @@ -27,32 +25,32 @@ Generator, Iterable, List, - Mapping, Optional, - Sequence, Tuple, Union, - cast, ) -import google.protobuf.message -import proto import requests -from google.protobuf import json_format import google.api_core.retry from google.api_core import exceptions from google.api_core.resumable_transfer import common, upload_state +from google.api_core.resumable_transfer.common import ( + ResumableUploadConfig, + _format_response_payload, +) _LOGGER = logging.getLogger(__name__) _DEFAULT_START_TIMEOUT = 60.0 # seconds for initial start request _monotonic_clock = time.monotonic -class _RecoveryRetransmit(Exception): - """Internal exception indicating state synchronization succeeded and chunk should retransmit.""" - - pass +def _get_buffer_size(stream: object) -> Optional[int]: + """Returns buffer size in bytes if stream exposes getbuffer(), else None.""" + getbuffer_fn = getattr(stream, "getbuffer", None) + if callable(getbuffer_fn): + return int(getbuffer_fn().nbytes) + return None class _IterableReader(io.BytesIO): @@ -61,7 +59,7 @@ class _IterableReader(io.BytesIO): Ensure that chunks are pulled lazily from the underlying iterator on each read() call rather than buffering the entire iterable into memory. Inherits from io.BytesIO so static type checkers recognize instances as - BinaryIO without casts. + BinaryIO natively. """ def __init__(self, iterable: Iterable[bytes]) -> None: @@ -94,61 +92,6 @@ def read(self, size: Optional[int] = -1) -> bytes: return result -@dataclasses.dataclass -class ResumableUploadConfig: - """Configuration options for a resumable upload. - - Attributes: - chunk_size: Size in bytes for each uploaded data chunk. Defaults to 10 MiB. - start_timeout: Local per-request timeout in seconds for start request. - start_retry: Custom retry policy for the start request. - stall_minimum_rate: Minimum transfer rate in bytes per second. Defaults to 64 KiB/s. - stall_timeout: Stall duration threshold in seconds. Defaults to 120s. - headers: Additional HTTP headers dispatched exclusively with start request. - deadline: Overall global deadline for the upload process. - timeout: Fallback per-request timeout. - retry: Fallback retry policy. - on_progress: Callback function receiving UploadProgress notifications. - response_type: Optional message class (proto.Message or google.protobuf.message.Message), - callable deserializer, or None to return raw response. - content_type: MIME type of the stream payload. - """ - - chunk_size: int = common.DEFAULT_CHUNK_SIZE - start_timeout: Optional[float] = None - start_retry: Optional[google.api_core.retry.Retry] = None - stall_minimum_rate: int = 64 * 1024 - stall_timeout: float = 120.0 - headers: Optional[Union[Mapping[str, str], Sequence[Tuple[str, str]]]] = None - deadline: Optional[datetime.datetime] = None - timeout: Optional[float] = None - retry: Optional[google.api_core.retry.Retry] = None - on_progress: Optional[Callable[[common.UploadProgress], None]] = None - response_type: Optional[Any] = None - content_type: Optional[str] = None - - def __post_init__(self) -> None: - """Normalizes fallback timeouts and retry policies.""" - if self.start_timeout is not None and self.timeout is None: - self.timeout = self.start_timeout - elif self.timeout is not None and self.start_timeout is None: - self.start_timeout = self.timeout - - if self.start_retry is not None and self.retry is None: - self.retry = self.start_retry - elif self.retry is not None and self.start_retry is None: - self.start_retry = self.retry - - @property - def start_headers(self) -> Optional[Sequence[Tuple[str, str]]]: - """Returns normalized additional headers for the start request.""" - if self.headers is None: - return None - if isinstance(self.headers, Mapping): - return list(self.headers.items()) - return list(self.headers) - - class ResumableUploadSession: """Manages the full lifecycle of a resumable upload session.""" @@ -185,7 +128,7 @@ def __init__( # Stall control tracking via monotonic clock self._aggregate_lag: float = 0.0 self._stall_timeout_started: Optional[float] = None - self._captured_progress: Optional[List[common.UploadProgress]] = None + self._needs_recovery: bool = False @property def upload_url(self) -> Optional[str]: @@ -238,11 +181,16 @@ def _enrich_exception(self, exc: BaseException) -> None: setattr(exc, "upload_url", self.upload_url) setattr(exc, "chunk_size", self.chunk_size) - def _notify_progress(self, state: common.ProgressState) -> None: - """Notifies progress with current upload status. + def _notify_progress( + self, + state: common.ProgressState, + progress_queue: Optional[List[common.UploadProgress]] = None, + ) -> None: + """Notifies registered callback and optional progress queue with current upload status. Args: state: ProgressState transition milestone. + progress_queue: Optional list buffering UploadProgress snapshots for generator consumers. """ if self.upload_url: progress = common.UploadProgress( @@ -252,20 +200,11 @@ def _notify_progress(self, state: common.ProgressState) -> None: total_bytes=self._state.total_bytes, state=state, ) - if self._captured_progress is not None: - self._captured_progress.append(progress) + if progress_queue is not None: + progress_queue.append(progress) if self._config.on_progress: self._config.on_progress(progress) - @contextlib.contextmanager - def _capture_progress(self) -> Generator[None, None, None]: - """Initializes self._captured_progress to buffer snapshots for generator consumers.""" - self._captured_progress = [] - try: - yield - finally: - self._captured_progress = None - def _get_deadline_remaining(self) -> Optional[float]: """Calculates remaining seconds until the configured upload deadline. @@ -302,12 +241,22 @@ def _get_start_timeout(self) -> float: return min(timeout, remaining) return timeout - def _get_retry_predicate(self) -> Callable[[Any], bool]: + def _get_retry_predicate(self, is_start: bool = False) -> Callable[[Any], bool]: """Returns a predicate function for determining if an exception is retryable. + Args: + is_start: If True, only transient status codes (RETRYABLE_STATUS_CODES) + are retried. If False (chunk transfer phase), state consistency + status codes (RECOVERABLE_STATUS_CODES) are also retried via recovery. + Returns: A callable accepting an exception and returning a boolean. """ + allowed_codes = ( + common.RETRYABLE_STATUS_CODES + if is_start + else (common.RECOVERABLE_STATUS_CODES + common.RETRYABLE_STATUS_CODES) + ) def should_retry(exc: Any) -> bool: if isinstance( @@ -316,6 +265,7 @@ def should_retry(exc: Any) -> bool: exceptions.DeadlineExceeded, exceptions.TransferStalledError, exceptions.UploadCancelledError, + exceptions.UnseekableStreamError, ), ): return False @@ -332,25 +282,60 @@ def should_retry(exc: Any) -> bool: ): return True if isinstance(exc, exceptions.GoogleAPICallError): - return exc.code in common.RETRYABLE_STATUS_CODES + return exc.code in allowed_codes return False return should_retry def _get_retry(self, is_start: bool = False) -> google.api_core.retry.Retry: - """Resolves retry policy for requests. + """Resolves unary Retry policy for start requests. Args: is_start: Whether this retry policy is for the start request. Returns: - Configured or default Retry instance. + Configured or default unary Retry instance. + """ + candidate = ( + self._config.start_retry + if is_start and self._config.start_retry + else self._config.retry + ) + if candidate is not None: + if isinstance(candidate, google.api_core.retry.Retry): + return candidate + return google.api_core.retry.Retry( + predicate=candidate._predicate, + initial=candidate._initial, + maximum=candidate._maximum, + multiplier=candidate._multiplier, + timeout=candidate._timeout, + on_error=candidate._on_error, + ) + return google.api_core.retry.Retry( + predicate=self._get_retry_predicate(is_start=is_start) + ) + + def _get_streaming_retry(self) -> google.api_core.retry.StreamingRetry: + """Resolves the StreamingRetry policy for the chunk upload generator. + + Returns: + Configured or default StreamingRetry instance. """ - if is_start and self._config.start_retry: - return self._config.start_retry if self._config.retry: - return self._config.retry - return google.api_core.retry.Retry(predicate=self._get_retry_predicate()) + if isinstance(self._config.retry, google.api_core.retry.StreamingRetry): + return self._config.retry + return google.api_core.retry.StreamingRetry( + predicate=self._config.retry._predicate, + initial=self._config.retry._initial, + maximum=self._config.retry._maximum, + multiplier=self._config.retry._multiplier, + timeout=self._config.retry._timeout, + on_error=self._config.retry._on_error, + ) + return google.api_core.retry.StreamingRetry( + predicate=self._get_retry_predicate(is_start=False) + ) def _compute_chunk_timeout(self, data_len: int) -> float: """Computes the dynamic per-attempt chunk timeout based on stall control and deadlines. @@ -417,7 +402,9 @@ def _update_stall_control( else: self._stall_timeout_started = None - def _reposition_stream_offset(self, stream: BinaryIO, received: int) -> int: + def _reposition_stream_offset( + self, stream: Union[BinaryIO, Iterable[bytes]], received: int + ) -> int: """Adjusts in-memory chunk buffer or seeks input stream to server offset. Args: @@ -440,14 +427,16 @@ def _reposition_stream_offset(self, stream: BinaryIO, received: int) -> int: return received self._buffered_chunk = None - if hasattr(stream, "seekable") and not stream.seekable(): + seekable_fn = getattr(stream, "seekable", None) + if callable(seekable_fn) and not seekable_fn(): raise exceptions.UnseekableStreamError( f"Stream is not seekable. Cannot recover upload to offset {received}.", upload_url=self.upload_url, chunk_size=self.chunk_size, ) try: - stream.seek(self._start_stream_offset + received) + seek_fn = getattr(stream, "seek") + seek_fn(self._start_stream_offset + received) except (OSError, AttributeError) as exc: raise exceptions.UnseekableStreamError( f"Failed to seek stream to offset {received}: {exc}", @@ -462,6 +451,7 @@ def initiate( transport: requests.Session, request_body: Union[str, bytes] = "", size: Optional[int] = None, + progress_queue: Optional[List[common.UploadProgress]] = None, ) -> str: """Initiates the upload session by sending the start command. @@ -469,6 +459,7 @@ def initiate( transport: The requests session. request_body: JSON payload for initial start request. size: Total size of payload in bytes, if known. + progress_queue: Optional list buffering UploadProgress snapshots. Returns: The upload session URL. @@ -492,167 +483,134 @@ def do_initiate() -> str: ) return session_url - session_url = self._get_retry(is_start=True)(do_initiate)() - self._notify_progress(common.ProgressState.STARTED) + retry = self._get_retry(is_start=True) + retryable_initiate = retry(do_initiate) + session_url = retryable_initiate() + self._notify_progress( + common.ProgressState.STARTED, progress_queue=progress_queue + ) return session_url def _transmit_chunk( - self, transport: requests.Session, stream: BinaryIO, size: Optional[int] + self, + transport: requests.Session, + stream: Union[BinaryIO, Iterable[bytes]], + size: Optional[int], + progress_queue: Optional[List[common.UploadProgress]] = None, ) -> requests.Response: - """Transmits the next data chunk with stall control and error recovery. + """Transmits a single data chunk attempt with stall control. Args: transport: The requests session. stream: The input data stream. size: Total size of the stream in bytes, if known. + progress_queue: Optional list buffering UploadProgress snapshots. Returns: The HTTP response for the transmitted chunk. """ + chunk_size = self._state.chunk_size + + # Retain active chunk in zero-copy buffer if not present. + # Ensure that EOF status (_buffered_chunk_is_last) is computed once + # when reading from the stream and preserved across _recover() retries. + # On partial server commit, _recover() slices _buffered_chunk in-place + # to the uncommitted tail. Preserving _buffered_chunk_is_last ensures + # that a sliced tail smaller than chunk_size is not prematurely + # treated as the final chunk when unread bytes remain in the stream. + if self._buffered_chunk is None: + read_fn = getattr(stream, "read") + raw_bytes = read_fn(chunk_size) + if not raw_bytes: + raw_bytes = b"" + self._buffered_chunk = memoryview(raw_bytes) + self._buffered_chunk_offset = self._state.bytes_uploaded + is_eof = len(raw_bytes) < chunk_size + if size is not None and self._state.bytes_uploaded + len(raw_bytes) >= size: + is_eof = True + self._buffered_chunk_is_last = is_eof + + data = self._buffered_chunk + data_len = len(data) + is_last = self._buffered_chunk_is_last + + method, url, headers, payload = self._state.build_chunk_request( + data=data, + is_last_chunk=is_last, + content_type=self._config.content_type, + ) - def do_transmit() -> requests.Response: - chunk_size = self._state.chunk_size - - # Retain active chunk in zero-copy buffer if not present. - # Ensure that EOF status (_buffered_chunk_is_last) is computed once - # when reading from the stream and preserved across _recover() retries. - # On partial server commit, _recover() slices _buffered_chunk in-place - # to the uncommitted tail. Preserving _buffered_chunk_is_last ensures - # that a sliced tail smaller than chunk_size is not prematurely - # treated as the final chunk when unread bytes remain in the stream. - if self._buffered_chunk is None: - raw_bytes = stream.read(chunk_size) - if not raw_bytes: - raw_bytes = b"" - self._buffered_chunk = memoryview(raw_bytes) - self._buffered_chunk_offset = self._state.bytes_uploaded - is_eof = len(raw_bytes) < chunk_size - if ( - size is not None - and self._state.bytes_uploaded + len(raw_bytes) >= size - ): - is_eof = True - self._buffered_chunk_is_last = is_eof - - data = self._buffered_chunk - data_len = len(data) - is_last = self._buffered_chunk_is_last - - method, url, headers, payload = self._state.build_chunk_request( - data=data, - is_last_chunk=is_last, - content_type=self._config.content_type, + per_attempt_timeout = self._compute_chunk_timeout(data_len) + try: + t_start = _monotonic_clock() + resp = transport.request( + method, + url, + data=payload, + headers=headers, + timeout=per_attempt_timeout, ) - - def do_http() -> requests.Response: - per_attempt_timeout = self._compute_chunk_timeout(data_len) - try: - resp = transport.request( - method, - url, - data=payload, - headers=headers, - timeout=per_attempt_timeout, - ) - except requests.exceptions.Timeout as exc: - if self._config.stall_minimum_rate and self._config.stall_timeout: - remaining = self._get_deadline_remaining() - if remaining is not None and remaining <= 0: - raise exceptions.DeadlineExceeded( - f"Resumable upload deadline {self._config.deadline} exceeded." - ) from exc - raise exceptions.TransferStalledError( - f"Upload stalled: chunk transfer timed out ({exc}).", - upload_url=self.upload_url, - chunk_size=self.chunk_size, - ) from exc - raise - if not resp.ok: - raise exceptions.from_http_response(resp) - return resp - - try: - t_start = _monotonic_clock() - resp = self._get_retry()(do_http)() - t_elapsed = _monotonic_clock() - t_start - - self._update_stall_control(data_len, t_start, t_elapsed) - self._state.process_chunk_response( - resp.status_code, resp.headers, data_len - ) - self._buffered_chunk = None - self._notify_progress( - common.ProgressState.FINALIZED - if self._state.finished - else common.ProgressState.UPLOADING - ) - return resp - except Exception as exc: - self._enrich_exception(exc) - if isinstance(exc, exceptions.DeadlineExceeded): - raise - if isinstance(exc, requests.exceptions.Timeout): - remaining = self._get_deadline_remaining() - if remaining is not None and remaining <= 0: - raise exceptions.DeadlineExceeded( - f"Resumable upload deadline {self._config.deadline} exceeded." - ) from exc - raise exceptions.TransferStalledError( - f"Upload stalled: chunk transfer timed out ({exc}).", - upload_url=self.upload_url, - chunk_size=self.chunk_size, - ) from exc - - is_recoverable = ( - isinstance(exc, exceptions.GoogleAPICallError) - and exc.code - in (common.RECOVERABLE_STATUS_CODES + common.RETRYABLE_STATUS_CODES) - ) or isinstance(exc, exceptions.MissingStatusHeaderError) - - if is_recoverable: - _LOGGER.info( - "Recoverable error %s during chunk upload. Querying server offset.", - exc, - ) - self._notify_progress(common.ProgressState.RECOVERING) - self._recover(transport, stream) - raise _RecoveryRetransmit() - raise - - recovery_loop = google.api_core.retry.Retry( - predicate=lambda e: isinstance(e, _RecoveryRetransmit) + t_elapsed = _monotonic_clock() - t_start + if not resp.ok: + raise exceptions.from_http_response(resp) + except requests.exceptions.Timeout as exc: + self._enrich_exception(exc) + self._get_deadline_remaining() + if self._config.stall_minimum_rate and self._config.stall_timeout: + raise exceptions.TransferStalledError( + f"Upload stalled: chunk transfer timed out ({exc}).", + upload_url=self.upload_url, + chunk_size=self.chunk_size, + ) from exc + raise + except Exception as exc: + self._enrich_exception(exc) + raise + + self._update_stall_control(data_len, t_start, t_elapsed) + self._state.process_chunk_response(resp.status_code, resp.headers, data_len) + self._buffered_chunk = None + self._notify_progress( + common.ProgressState.FINALIZED + if self._state.finished + else common.ProgressState.UPLOADING, + progress_queue=progress_queue, ) - return recovery_loop(do_transmit)() + return resp - def _recover(self, transport: requests.Session, stream: BinaryIO) -> int: + def _recover( + self, + transport: requests.Session, + stream: Union[BinaryIO, Iterable[bytes]], + progress_queue: Optional[List[common.UploadProgress]] = None, + ) -> requests.Response: """Queries server for committed byte offset and adjusts buffer / stream. Args: transport: The requests session. stream: The input data stream. + progress_queue: Optional list buffering UploadProgress snapshots. Returns: - The confirmed server byte offset. + The HTTP response for the status query. Raises: exceptions.UnseekableStreamError: If server offset precedes buffer and stream cannot be rewound. exceptions.GoogleAPICallError: If query request fails on the server. """ method, url, headers, payload = self._state.build_query_request() - - def do_query() -> requests.Response: - timeout = self._get_start_timeout() - resp = transport.request( - method, url, data=payload, headers=headers, timeout=timeout - ) - if not resp.ok: - raise exceptions.from_http_response(resp) - return resp - - resp = self._get_retry()(do_query)() + timeout = self._get_start_timeout() + resp = transport.request( + method, url, data=payload, headers=headers, timeout=timeout + ) + if not resp.ok: + raise exceptions.from_http_response(resp) received = self._state.process_query_response(resp.status_code, resp.headers) - self._notify_progress(common.ProgressState.OFFSET_RECEIVED) - return self._reposition_stream_offset(stream, received) + self._notify_progress( + common.ProgressState.OFFSET_RECEIVED, progress_queue=progress_queue + ) + self._reposition_stream_offset(stream, received) + return resp def cancel(self, transport: Optional[requests.Session] = None) -> None: """Cancels the resumable upload session. @@ -675,8 +633,9 @@ def cancel(self, transport: Optional[requests.Session] = None) -> None: def _transmit_all_chunks( self, transport: requests.Session, - stream_obj: BinaryIO, + stream_obj: Union[BinaryIO, Iterable[bytes]], computed_size: Optional[int], + progress_queue: Optional[List[common.UploadProgress]] = None, ) -> Generator[common.UploadProgress, None, None]: """Transmits chunks until transfer completes, yielding buffered progress updates. @@ -684,6 +643,7 @@ def _transmit_all_chunks( transport: The requests session. stream_obj: Binary stream yielding upload chunks. computed_size: Total payload size in bytes if known. + progress_queue: Optional list buffering UploadProgress snapshots. Yields: UploadProgress snapshots for each transmission milestone. @@ -691,16 +651,76 @@ def _transmit_all_chunks( Raises: ValueError: If upload concludes without a server response. """ - if self._captured_progress: - while self._captured_progress: - yield self._captured_progress.pop(0) + if progress_queue is None: + progress_queue = [] + + while progress_queue: + yield progress_queue.pop(0) + + final_resp: Optional[requests.Response] = None + + def attempt_stream() -> Generator[common.UploadProgress, None, None]: + nonlocal final_resp + if self._needs_recovery: + _LOGGER.info( + "Recoverable error during chunk upload. Querying server offset." + ) + self._notify_progress( + common.ProgressState.RECOVERING, + progress_queue=progress_queue, + ) + recover_resp = self._recover( + transport, stream_obj, progress_queue=progress_queue + ) + if self._state.finished: + final_resp = recover_resp + self._needs_recovery = False + while progress_queue: + yield progress_queue.pop(0) - final_resp = None - while not self._state.finished and not self._state.invalid: - final_resp = self._transmit_chunk(transport, stream_obj, computed_size) - if self._captured_progress: - while self._captured_progress: - yield self._captured_progress.pop(0) + while not self._state.finished and not self._state.invalid: + try: + final_resp = self._transmit_chunk( + transport, + stream_obj, + computed_size, + progress_queue=progress_queue, + ) + except Exception as exc: + is_recoverable = ( + isinstance(exc, exceptions.GoogleAPICallError) + and exc.code + in ( + common.RECOVERABLE_STATUS_CODES + + common.RETRYABLE_STATUS_CODES + ) + ) or isinstance( + exc, + ( + exceptions.MissingStatusHeaderError, + requests.exceptions.ConnectionError, + requests.exceptions.ChunkedEncodingError, + requests.exceptions.Timeout, + ), + ) + if is_recoverable: + self._needs_recovery = True + raise + while progress_queue: + yield progress_queue.pop(0) + + try: + retry = self._get_streaming_retry() + retryable_stream = retry(attempt_stream) + yield from retryable_stream() + except requests.exceptions.Timeout as exc: + self._enrich_exception(exc) + self._get_deadline_remaining() + raise exceptions.TransferStalledError( + f"Upload stalled: chunk transfer timed out ({exc}).", + upload_url=self.upload_url, + chunk_size=self.chunk_size, + ) from exc if final_resp is None: raise ValueError("Upload completed without receiving a final response.") @@ -758,16 +778,21 @@ def iter_upload( GoogleAPICallError: If an unrecoverable API error occurs. """ sess = self._get_transport(transport) - with self._capture_progress(): - try: - stream_obj, computed_size = self._prepare_stream(stream, size) - self.initiate( - transport=sess, request_body=request_body, size=computed_size - ) - yield from self._transmit_all_chunks(sess, stream_obj, computed_size) - except Exception as exc: - self._enrich_exception(exc) - raise + progress_queue: List[common.UploadProgress] = [] + try: + stream_obj, computed_size = self._prepare_stream(stream, size) + self.initiate( + transport=sess, + request_body=request_body, + size=computed_size, + progress_queue=progress_queue, + ) + yield from self._transmit_all_chunks( + sess, stream_obj, computed_size, progress_queue=progress_queue + ) + except Exception as exc: + self._enrich_exception(exc) + raise def resume( self, @@ -838,56 +863,63 @@ def iter_resume( self._state._chunk_size = chunk_size self._state._resumable_url = actual_url - with self._capture_progress(): - try: - stream_obj, computed_size = self._prepare_stream(stream, size) - self._recover(sess, stream_obj) - yield from self._transmit_all_chunks(sess, stream_obj, computed_size) - except Exception as exc: - self._enrich_exception(exc) - raise + progress_queue: List[common.UploadProgress] = [] + try: + stream_obj, computed_size = self._prepare_stream(stream, size) + self._recover(sess, stream_obj, progress_queue=progress_queue) + yield from self._transmit_all_chunks( + sess, stream_obj, computed_size, progress_queue=progress_queue + ) + except Exception as exc: + self._enrich_exception(exc) + raise def _prepare_stream( self, stream: Union[BinaryIO, bytes, Iterable[bytes]], size: Optional[int] - ) -> Tuple[BinaryIO, Optional[int]]: - """Normalizes stream input into a BinaryIO object and determines stream length. + ) -> Tuple[Union[BinaryIO, Iterable[bytes]], Optional[int]]: + """Normalizes stream input into a readable stream object and determines stream length. Args: stream: Input stream, bytes, or iterable of bytes. size: Explicit total size in bytes, if known. Returns: - Tuple of (prepared BinaryIO stream, computed total size). + Tuple of (prepared stream object, computed total size). """ computed_size = size if isinstance(stream, (str, dict)): raise TypeError(f"Unsupported stream type: {type(stream)}") if isinstance(stream, bytes): - stream_obj: BinaryIO = io.BytesIO(stream) + stream_obj: Union[BinaryIO, Iterable[bytes]] = io.BytesIO(stream) if computed_size is None: computed_size = len(stream) elif not hasattr(stream, "read") and isinstance(stream, Iterable): stream_obj = _IterableReader(stream) elif hasattr(stream, "read"): - stream_obj = cast(BinaryIO, stream) + stream_obj = stream if computed_size is None: - if hasattr(stream_obj, "getbuffer"): - computed_size = stream_obj.getbuffer().nbytes - elif ( - hasattr(stream_obj, "seekable") - and stream_obj.seekable() - and hasattr(stream_obj, "tell") - ): - cur = stream_obj.tell() - stream_obj.seek(0, io.SEEK_END) - computed_size = stream_obj.tell() - cur - stream_obj.seek(cur) + computed_size = _get_buffer_size(stream_obj) + seekable_fn = getattr(stream_obj, "seekable", None) + tell_fn = getattr(stream_obj, "tell", None) + seek_fn = getattr(stream_obj, "seek", None) + if ( + computed_size is None + and callable(seekable_fn) + and seekable_fn() + and callable(tell_fn) + and callable(seek_fn) + ): + cur = tell_fn() + seek_fn(0, io.SEEK_END) + computed_size = tell_fn() - cur + seek_fn(cur) else: raise TypeError(f"Unsupported stream type: {type(stream)}") - if hasattr(stream_obj, "tell"): + tell_fn = getattr(stream_obj, "tell", None) + if callable(tell_fn): try: - self._start_stream_offset = stream_obj.tell() + self._start_stream_offset = tell_fn() except (OSError, AttributeError): self._start_stream_offset = 0 @@ -903,45 +935,3 @@ def _format_response(self, response: requests.Response) -> Any: Deserialized protobuf message or the raw response object. """ return _format_response_payload(response, self._config.response_type) - - -def _format_response_payload( - response: Union[Any, bytes], - response_type: Optional[Any], -) -> Any: - """Formats raw response or bytes into protobuf or proto-plus message type if configured. - - Args: - response: Raw HTTP response object or response body bytes. - response_type: Deserializer callable, proto.Message class, or - google.protobuf.message.Message class or instance. - - Returns: - Deserialized protobuf message or the raw response object / bytes. - """ - if response_type is None: - return response - - content: bytes - if isinstance(response, bytes): - content = response - elif hasattr(response, "content"): - content = response.content - else: - content = bytes(response) - - if isinstance(response_type, type) and issubclass(response_type, proto.Message): - return cast(Any, response_type).from_json(content, ignore_unknown_fields=True) - if isinstance(response_type, type) and issubclass( - response_type, google.protobuf.message.Message - ): - instance = response_type() - return json_format.Parse(content, instance, ignore_unknown_fields=True) - if isinstance(response_type, google.protobuf.message.Message): - return json_format.Parse(content, response_type, ignore_unknown_fields=True) - if hasattr(response_type, "from_json") and callable(response_type.from_json): - return response_type.from_json(content) - if callable(response_type): - return response_type(content) - - return response diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 3b122b38b47d..870b351f8b03 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -22,6 +22,7 @@ import time from typing import ( Any, + AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, @@ -40,11 +41,12 @@ try: import aiohttp except ImportError: # pragma: NO COVER - aiohttp = None # type: ignore + pass +import google.api_core.retry from google.api_core import exceptions from google.api_core.resumable_transfer import common, upload_state -from google.api_core.resumable_transfer.upload import ( +from google.api_core.resumable_transfer.common import ( ResumableUploadConfig, _format_response_payload, ) @@ -54,13 +56,16 @@ _DONE_SENTINEL = object() _monotonic_clock = time.monotonic -ResponseProto = TypeVar("ResponseProto") +def _get_buffer_size(stream: object) -> Optional[int]: + """Returns buffer size in bytes if stream exposes getbuffer(), else None.""" + getbuffer_fn = getattr(stream, "getbuffer", None) + if callable(getbuffer_fn): + return int(getbuffer_fn().nbytes) + return None -class _AsyncRecoveryRetransmit(Exception): - """Internal exception indicating state synchronization succeeded and chunk should retransmit.""" - pass +ResponseProto = TypeVar("ResponseProto") class AsyncUploadOperation(Generic[ResponseProto], Awaitable[ResponseProto]): @@ -165,6 +170,7 @@ def __init__( # Stall control tracking via monotonic clock self._aggregate_lag: float = 0.0 self._stall_timeout_started: Optional[float] = None + self._needs_recovery: bool = False @property def upload_url(self) -> Optional[str]: @@ -197,7 +203,7 @@ def _ensure_aiohttp(self) -> None: Raises: ImportError: If aiohttp is not installed. """ - if aiohttp is None: + if globals().get("aiohttp") is None: raise ImportError( "The aiohttp library is required to use AsyncResumableUploadSession. " "Please install google-api-core[async_rest]." @@ -273,44 +279,101 @@ def _get_start_timeout(self) -> float: return min(timeout, remaining) return timeout - async def _async_retry( - self, coro_fn: Callable[[], Awaitable[Any]], max_attempts: int = 4 - ) -> Any: - """Executes an asynchronous callable with exponential backoff retry logic. + def _get_retry_predicate(self, is_start: bool = False) -> Callable[[Any], bool]: + """Returns a predicate function for determining if an exception is retryable. Args: - coro_fn: Asynchronous nullary function to invoke and retry. - max_attempts: Maximum retry attempts before propagating failure. + is_start: If True, only transient status codes (RETRYABLE_STATUS_CODES) + are retried. If False (chunk transfer phase), state consistency + status codes (RECOVERABLE_STATUS_CODES) are also retried via recovery. Returns: - The successful return value of coro_fn. + A callable accepting an exception and returning a boolean. """ - delay = 1.0 - multiplier = 2.0 - max_delay = 60.0 - for attempt in range(max_attempts): - try: - return await coro_fn() - except ( - exceptions.DeadlineExceeded, - exceptions.TransferStalledError, - exceptions.UploadCancelledError, + allowed_codes = ( + common.RETRYABLE_STATUS_CODES + if is_start + else (common.RECOVERABLE_STATUS_CODES + common.RETRYABLE_STATUS_CODES) + ) + + def should_retry(exc: Any) -> bool: + if isinstance( + exc, + ( + exceptions.DeadlineExceeded, + exceptions.TransferStalledError, + exceptions.UploadCancelledError, + exceptions.UnseekableStreamError, + ), ): - raise - except exceptions.MissingStatusHeaderError: - if attempt == max_attempts - 1: - raise - except exceptions.GoogleAPICallError as exc: - if exc.code not in common.RETRYABLE_STATUS_CODES: - raise - if attempt == max_attempts - 1: - raise - except Exception: - if attempt == max_attempts - 1: - raise + return False + if isinstance(exc, exceptions.MissingStatusHeaderError): + return True + if globals().get("aiohttp") is not None and isinstance( + exc, aiohttp.ClientError + ): + return True + if isinstance(exc, exceptions.GoogleAPICallError): + return exc.code in allowed_codes + return False + + return should_retry + + def _get_async_retry( + self, is_start: bool = False + ) -> google.api_core.retry.AsyncRetry: + """Resolves unary AsyncRetry policy for start requests. + + Args: + is_start: Whether this retry policy is for the start request. + + Returns: + Configured or default unary AsyncRetry instance. + """ + candidate = ( + self._config.start_retry + if is_start and self._config.start_retry + else self._config.retry + ) + if candidate is not None: + if isinstance(candidate, google.api_core.retry.AsyncRetry): + return candidate + return google.api_core.retry.AsyncRetry( + predicate=candidate._predicate, + initial=candidate._initial, + maximum=candidate._maximum, + multiplier=candidate._multiplier, + timeout=candidate._timeout, + on_error=candidate._on_error, + ) + return google.api_core.retry.AsyncRetry( + predicate=self._get_retry_predicate(is_start=is_start) + ) - await asyncio.sleep(delay) - delay = min(delay * multiplier, max_delay) + def _get_async_streaming_retry( + self, + ) -> google.api_core.retry.AsyncStreamingRetry: + """Resolves the AsyncStreamingRetry policy for the chunk upload generator. + + Returns: + Configured or default AsyncStreamingRetry instance. + """ + if self._config.retry: + if isinstance( + self._config.retry, google.api_core.retry.AsyncStreamingRetry + ): + return self._config.retry + return google.api_core.retry.AsyncStreamingRetry( + predicate=self._config.retry._predicate, + initial=self._config.retry._initial, + maximum=self._config.retry._maximum, + multiplier=self._config.retry._multiplier, + timeout=self._config.retry._timeout, + on_error=self._config.retry._on_error, + ) + return google.api_core.retry.AsyncStreamingRetry( + predicate=self._get_retry_predicate(is_start=False) + ) async def initiate( self, @@ -359,7 +422,9 @@ async def do_initiate(): ) return session_url - session_url = await self._async_retry(do_initiate) + retry = self._get_async_retry(is_start=True) + retryable_initiate = retry(do_initiate) + session_url = await retryable_initiate() self._notify_progress(common.ProgressState.STARTED, progress_queue) return session_url @@ -369,7 +434,6 @@ async def _transmit_chunk( reader_fn: Callable[[int], Awaitable[bytes]], size: Optional[int], progress_queue: Optional[asyncio.Queue] = None, - stream_obj: Any = None, ) -> Tuple[int, Mapping[str, str], bytes]: """Transmits the next data chunk asynchronously with stall control. @@ -378,7 +442,6 @@ async def _transmit_chunk( reader_fn: Async callable returning chunk bytes. size: Total stream size in bytes, if known. progress_queue: Optional queue to receive progress updates. - stream_obj: Underlying stream object for recovery seeking. Returns: Tuple of (status code, headers mapping, response body bytes). @@ -386,205 +449,161 @@ async def _transmit_chunk( Raises: TransferStalledError: If chunk transfer throughput stalls. DeadlineExceeded: If upload deadline is reached. - GoogleAPICallError: If chunk upload encounters an unrecoverable error. + GoogleAPICallError: If chunk upload encounters an error. """ + chunk_size = self._state.chunk_size + + # Retain active chunk in zero-copy buffer if not present. + # Ensure that EOF status (_buffered_chunk_is_last) is computed once + # when reading from the stream and preserved across _recover() retries. + # On partial server commit, _recover() slices _buffered_chunk in-place + # to the uncommitted tail. Preserving _buffered_chunk_is_last ensures + # that a sliced tail smaller than chunk_size is not prematurely + # treated as the final chunk when unread bytes remain in the stream. + if self._buffered_chunk is None: + raw_bytes = await reader_fn(chunk_size) + if not raw_bytes: + raw_bytes = b"" + self._buffered_chunk = memoryview(raw_bytes) + self._buffered_chunk_offset = self._state.bytes_uploaded + is_eof = len(raw_bytes) < chunk_size + if size is not None and self._state.bytes_uploaded + len(raw_bytes) >= size: + is_eof = True + self._buffered_chunk_is_last = is_eof + + data = self._buffered_chunk + data_len = len(data) + is_last = self._buffered_chunk_is_last + + method, url, headers, payload = self._state.build_chunk_request( + data=data, + is_last_chunk=is_last, + content_type=self._config.content_type, + ) - async def do_transmit(): - chunk_size = self._state.chunk_size - - # Retain active chunk in zero-copy buffer if not present. - # Ensure that EOF status (_buffered_chunk_is_last) is computed once - # when reading from the stream and preserved across _recover() retries. - # On partial server commit, _recover() slices _buffered_chunk in-place - # to the uncommitted tail. Preserving _buffered_chunk_is_last ensures - # that a sliced tail smaller than chunk_size is not prematurely - # treated as the final chunk when unread bytes remain in the stream. - if self._buffered_chunk is None: - raw_bytes = await reader_fn(chunk_size) - if not raw_bytes: - raw_bytes = b"" - self._buffered_chunk = memoryview(raw_bytes) - self._buffered_chunk_offset = self._state.bytes_uploaded - is_eof = len(raw_bytes) < chunk_size - if ( - size is not None - and self._state.bytes_uploaded + len(raw_bytes) >= size - ): - is_eof = True - self._buffered_chunk_is_last = is_eof - - data = self._buffered_chunk - data_len = len(data) - is_last = self._buffered_chunk_is_last - - method, url, headers, payload = self._state.build_chunk_request( - data=data, - is_last_chunk=is_last, - content_type=self._config.content_type, - ) + rate = self._config.stall_minimum_rate + expected_sec = data_len / rate if rate > 0 else 60.0 + next_chunk_timeout = max( + 1.0, + expected_sec - self._aggregate_lag + self._config.stall_timeout, + ) + per_attempt_timeout = max(5.0, min(next_chunk_timeout, 2.0 * expected_sec)) - async def do_http(): - rate = self._config.stall_minimum_rate - expected_sec = data_len / rate if rate > 0 else 60.0 - next_chunk_timeout = max( - 1.0, - expected_sec - self._aggregate_lag + self._config.stall_timeout, - ) - per_attempt_timeout = max( - 5.0, min(next_chunk_timeout, 2.0 * expected_sec) - ) + if self._config.timeout: + per_attempt_timeout = min(self._config.timeout, per_attempt_timeout) - if self._config.timeout: - per_attempt_timeout = min(self._config.timeout, per_attempt_timeout) + remaining = self._get_deadline_remaining() + if remaining is not None: + per_attempt_timeout = min(per_attempt_timeout, remaining) + client_timeout = aiohttp.ClientTimeout(total=per_attempt_timeout) + t_start = _monotonic_clock() + try: + async with transport.request( + method, + url, + data=payload, + headers=headers, + timeout=client_timeout, + ) as resp: + resp_headers = dict(resp.headers) + resp_body = await resp.read() + if resp.status not in (200, 201): + raise exceptions.from_http_status( + resp.status, resp_body.decode("utf-8", errors="replace") + ) + status_code = resp.status + t_elapsed = _monotonic_clock() - t_start + except Exception as exc: + self._enrich_exception(exc) + if isinstance( + exc, + ( + asyncio.TimeoutError, + aiohttp.ServerTimeoutError, + ), + ): remaining = self._get_deadline_remaining() - if remaining is not None: - per_attempt_timeout = min(per_attempt_timeout, remaining) - - client_timeout = aiohttp.ClientTimeout(total=per_attempt_timeout) - try: - async with transport.request( - method, - url, - data=payload, - headers=headers, - timeout=client_timeout, - ) as resp: - resp_headers = dict(resp.headers) - resp_body = await resp.read() - if resp.status not in (200, 201): - raise exceptions.from_http_status( - resp.status, resp_body.decode("utf-8", errors="replace") - ) - return resp.status, resp_headers, resp_body - except asyncio.TimeoutError as exc: - if self._config.stall_minimum_rate and self._config.stall_timeout: - remaining = self._get_deadline_remaining() - if remaining is not None and remaining <= 0: - raise exceptions.DeadlineExceeded( - f"Resumable upload deadline {self._config.deadline} exceeded." - ) from exc - raise exceptions.TransferStalledError( - f"Upload stalled: chunk transfer timed out ({exc}).", - upload_url=self.upload_url, - chunk_size=self.chunk_size, - ) from exc - raise - - try: - t_start = _monotonic_clock() - status_code, resp_headers, resp_body = await self._async_retry(do_http) - t_elapsed = _monotonic_clock() - t_start - - # Evaluate stall control lag & timer + if remaining is not None and remaining <= 0: + raise exceptions.DeadlineExceeded( + "Resumable upload deadline exceeded during chunk transfer." + ) from exc if self._config.stall_minimum_rate and self._config.stall_timeout: - rate = self._config.stall_minimum_rate - expected_sec = data_len / rate if rate > 0 else 0.0 - current_lag = t_elapsed - expected_sec - self._aggregate_lag = max(0.0, self._aggregate_lag + current_lag) - if self._aggregate_lag > 0.0: - if self._stall_timeout_started is None: - self._stall_timeout_started = t_start - if ( - _monotonic_clock() - self._stall_timeout_started - >= self._config.stall_timeout - ): - self._get_deadline_remaining() - raise exceptions.TransferStalledError( - f"Upload stalled: transfer rate remained below {rate} bytes/s for longer than {self._config.stall_timeout}s.", - upload_url=self.upload_url, - chunk_size=self.chunk_size, - ) - else: - self._stall_timeout_started = None - - self._state.process_chunk_response(status_code, resp_headers, data_len) - self._buffered_chunk = None - self._notify_progress( - common.ProgressState.FINALIZED - if self._state.finished - else common.ProgressState.UPLOADING, - progress_queue, - ) - return status_code, resp_headers, resp_body - except Exception as exc: - self._enrich_exception(exc) - if isinstance(exc, exceptions.DeadlineExceeded): - raise - if isinstance( - exc, - ( - asyncio.TimeoutError, - aiohttp.ServerTimeoutError if aiohttp else (), - ), - ): - remaining = self._get_deadline_remaining() - if remaining is not None and remaining <= 0: - raise exceptions.DeadlineExceeded( - f"Resumable upload deadline {self._config.deadline} exceeded." - ) from exc raise exceptions.TransferStalledError( f"Upload stalled: chunk transfer timed out ({exc}).", upload_url=self.upload_url, chunk_size=self.chunk_size, ) from exc - - is_recoverable = ( - isinstance(exc, exceptions.GoogleAPICallError) - and exc.code - in (common.RECOVERABLE_STATUS_CODES + common.RETRYABLE_STATUS_CODES) - ) or isinstance(exc, exceptions.MissingStatusHeaderError) - - if is_recoverable: - _LOGGER.info( - "Recoverable error %s during async chunk upload. Querying server offset.", - exc, - ) - self._notify_progress( - common.ProgressState.RECOVERING, progress_queue + raise + + # Evaluate stall control lag & timer + if self._config.stall_minimum_rate and self._config.stall_timeout: + rate = self._config.stall_minimum_rate + expected_sec = data_len / rate if rate > 0 else 0.0 + current_lag = t_elapsed - expected_sec + self._aggregate_lag = max(0.0, self._aggregate_lag + current_lag) + if self._aggregate_lag > 0.0: + if self._stall_timeout_started is None: + self._stall_timeout_started = t_start + if ( + _monotonic_clock() - self._stall_timeout_started + >= self._config.stall_timeout + ): + self._get_deadline_remaining() + raise exceptions.TransferStalledError( + f"Upload stalled: transfer rate remained below {rate} bytes/s for longer than {self._config.stall_timeout}s.", + upload_url=self.upload_url, + chunk_size=self.chunk_size, ) - await self._recover(transport, stream_obj) - raise _AsyncRecoveryRetransmit() - raise + else: + self._stall_timeout_started = None - while True: - try: - return await do_transmit() - except _AsyncRecoveryRetransmit: - continue + self._state.process_chunk_response(status_code, resp_headers, data_len) + self._buffered_chunk = None + self._notify_progress( + common.ProgressState.FINALIZED + if self._state.finished + else common.ProgressState.UPLOADING, + progress_queue, + ) + return status_code, resp_headers, resp_body - async def _recover(self, transport: Any, stream_obj: Any = None) -> int: + async def _recover( + self, + transport: Any, + stream_obj: Optional[object] = None, + progress_queue: Optional[asyncio.Queue] = None, + ) -> Tuple[int, Mapping[str, str], bytes]: """Queries server for committed byte offset and adjusts buffer. Args: transport: The aiohttp client session. stream_obj: Underlying stream object to rewind if seekable. + progress_queue: Optional queue to receive progress updates. Returns: - The confirmed server byte offset. + Tuple of (status code, headers mapping, response body bytes). Raises: exceptions.UnseekableStreamError: If server offset precedes buffer and stream cannot be rewound. exceptions.GoogleAPICallError: If query request fails on the server. """ method, url, headers, payload = self._state.build_query_request() + timeout_sec = self._get_start_timeout() + client_timeout = aiohttp.ClientTimeout(total=timeout_sec) + async with transport.request( + method, url, data=payload, headers=headers, timeout=client_timeout + ) as resp: + resp_headers = dict(resp.headers) + body = await resp.read() + if resp.status not in (200, 201): + raise exceptions.from_http_status( + resp.status, body.decode("utf-8", errors="replace") + ) + status_code = resp.status - async def do_query(): - timeout_sec = self._get_start_timeout() - client_timeout = aiohttp.ClientTimeout(total=timeout_sec) - async with transport.request( - method, url, data=payload, headers=headers, timeout=client_timeout - ) as resp: - resp_headers = dict(resp.headers) - body = await resp.read() - if resp.status not in (200, 201): - raise exceptions.from_http_status( - resp.status, body.decode("utf-8", errors="replace") - ) - return resp.status, resp_headers - - status_code, resp_headers = await self._async_retry(do_query) received = self._state.process_query_response(status_code, resp_headers) + self._notify_progress(common.ProgressState.OFFSET_RECEIVED, progress_queue) if self._buffered_chunk is not None: chunk_start = self._buffered_chunk_offset @@ -593,19 +612,21 @@ async def do_query(): discard_len = received - chunk_start self._buffered_chunk = self._buffered_chunk[discard_len:] self._buffered_chunk_offset = received - return received + return status_code, resp_headers, body self._buffered_chunk = None - if stream_obj is not None and hasattr(stream_obj, "seek"): - if hasattr(stream_obj, "seekable") and not stream_obj.seekable(): + seek_fn = getattr(stream_obj, "seek", None) + if callable(seek_fn): + seekable_fn = getattr(stream_obj, "seekable", None) + if callable(seekable_fn) and not seekable_fn(): raise exceptions.UnseekableStreamError( f"Stream is not seekable. Cannot recover upload to offset {received}.", upload_url=self.upload_url, chunk_size=self.chunk_size, ) try: - stream_obj.seek(self._start_stream_offset + received) - return received + seek_fn(self._start_stream_offset + received) + return status_code, resp_headers, body except (OSError, AttributeError) as exc: raise exceptions.UnseekableStreamError( f"Failed to seek stream to offset {received}: {exc}", @@ -647,6 +668,88 @@ async def cancel(self, transport: Optional[Any] = None) -> None: ) self._state.process_cancel_response(resp.status, resp_headers) + async def _transmit_all_chunks( + self, + transport: Any, + reader_fn: Callable[[int], Awaitable[bytes]], + computed_size: Optional[int], + progress_queue: Optional[asyncio.Queue] = None, + stream_obj: Optional[object] = None, + ) -> Optional[Tuple[int, Mapping[str, str], bytes]]: + """Transmits chunks until completion using a single outer AsyncStreamingRetry coordinator. + + Args: + transport: The aiohttp client session. + reader_fn: Async callable returning chunk bytes. + computed_size: Total stream size in bytes, if known. + progress_queue: Optional queue receiving UploadProgress snapshots. + stream_obj: Underlying stream object for recovery seeking. + + Returns: + Tuple of (status code, headers, body bytes) of the final server response, or None. + """ + final_resp_tuple: Optional[Tuple[int, Mapping[str, str], bytes]] = None + + async def attempt_stream() -> AsyncGenerator[None, None]: + nonlocal final_resp_tuple + if self._needs_recovery: + _LOGGER.info( + "Recoverable error during async chunk upload. Querying server offset." + ) + self._notify_progress(common.ProgressState.RECOVERING, progress_queue) + recover_tuple = await self._recover( + transport, stream_obj, progress_queue=progress_queue + ) + if self._state.finished: + final_resp_tuple = recover_tuple + self._needs_recovery = False + yield + + while not self._state.finished and not self._state.invalid: + try: + final_resp_tuple = await self._transmit_chunk( + transport, reader_fn, computed_size, progress_queue + ) + except Exception as exc: + is_recoverable = ( + isinstance(exc, exceptions.GoogleAPICallError) + and exc.code + in ( + common.RECOVERABLE_STATUS_CODES + + common.RETRYABLE_STATUS_CODES + ) + ) or isinstance( + exc, + ( + exceptions.MissingStatusHeaderError, + aiohttp.ClientError, + ), + ) + if is_recoverable: + self._needs_recovery = True + raise + yield + + try: + retry = self._get_async_streaming_retry() + retryable_stream = retry(attempt_stream) + stream_gen = await retryable_stream() + async for _ in stream_gen: + pass + except ( + asyncio.TimeoutError, + aiohttp.ServerTimeoutError, + ) as exc: + self._enrich_exception(exc) + self._get_deadline_remaining() + raise exceptions.TransferStalledError( + f"Upload stalled: chunk transfer timed out ({exc}).", + upload_url=self.upload_url, + chunk_size=self.chunk_size, + ) from exc + + return final_resp_tuple + def upload( self, stream: Union[AsyncIterable[bytes], BinaryIO, bytes, Iterable[bytes]], @@ -685,11 +788,9 @@ async def _run(): progress_queue=progress_queue, ) - final_resp_tuple = None - while not self._state.finished and not self._state.invalid: - final_resp_tuple = await self._transmit_chunk( - sess, reader_fn, computed_size, progress_queue, stream_obj - ) + final_resp_tuple = await self._transmit_all_chunks( + sess, reader_fn, computed_size, progress_queue, stream_obj + ) if final_resp_tuple is None: raise ValueError( @@ -747,16 +848,11 @@ def resume( async def _run(): try: - await self._recover(sess, stream_obj) - self._notify_progress( - common.ProgressState.OFFSET_RECEIVED, progress_queue - ) + await self._recover(sess, stream_obj, progress_queue=progress_queue) - final_resp_tuple = None - while not self._state.finished and not self._state.invalid: - final_resp_tuple = await self._transmit_chunk( - sess, reader_fn, computed_size, progress_queue, stream_obj - ) + final_resp_tuple = await self._transmit_all_chunks( + sess, reader_fn, computed_size, progress_queue, stream_obj + ) if final_resp_tuple is None: raise ValueError( @@ -781,7 +877,11 @@ def _prepare_async_reader( self, stream: Union[AsyncIterable[bytes], BinaryIO, bytes, Iterable[bytes]], size: Optional[int], - ) -> Tuple[Callable[[int], Awaitable[bytes]], Optional[int], Any]: + ) -> Tuple[ + Callable[[int], Awaitable[bytes]], + Optional[int], + Optional[object], + ]: """Creates an asynchronous byte reader and determines stream length. Args: @@ -809,29 +909,31 @@ async def reader(n: int) -> bytes: return reader, computed_size, bytes_io - if hasattr(stream, "read") and inspect.iscoroutinefunction(stream.read): - # Native async reader (e.g. asyncio.StreamReader) - async def reader(n: int) -> bytes: - return await stream.read(n) # type: ignore + read_fn = getattr(stream, "read", None) + if callable(read_fn): + if inspect.iscoroutinefunction(read_fn): + # Native async reader (e.g. asyncio.StreamReader) + async def reader(n: int) -> bytes: + return await read_fn(n) - return reader, computed_size, stream + return reader, computed_size, stream - if hasattr(stream, "read"): - # Synchronous binary stream: offload blocking reads to worker thread - sync_stream: Any = stream - if computed_size is None and hasattr(sync_stream, "getbuffer"): - computed_size = sync_stream.getbuffer().nbytes + # Synchronous binary stream (e.g. io.BytesIO or open file handle): + # offload blocking reads to worker thread via asyncio.to_thread. + if computed_size is None: + computed_size = _get_buffer_size(stream) - if hasattr(sync_stream, "tell"): + tell_fn = getattr(stream, "tell", None) + if callable(tell_fn): try: - self._start_stream_offset = sync_stream.tell() + self._start_stream_offset = tell_fn() except (OSError, AttributeError): self._start_stream_offset = 0 async def reader(n: int) -> bytes: - return await asyncio.to_thread(sync_stream.read, n) + return await asyncio.to_thread(read_fn, n) - return reader, computed_size, sync_stream + return reader, computed_size, stream if hasattr(stream, "__aiter__"): # Native AsyncIterable[bytes] diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index dd7b74465eef..e690094cae68 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -467,8 +467,9 @@ async def test_async_stream_types_sync_iterable() -> None: def test_async_stream_types_unsupported_raises() -> None: """Verifies that passing an unsupported stream type raises TypeError.""" session = AsyncResumableUploadSession(transport=DummyAsyncSession()) + prepare_reader = getattr(session, "_prepare_async_reader") with pytest.raises(TypeError, match="Unsupported stream type"): - session._prepare_async_reader(stream=12345, size=10) # type: ignore + prepare_reader(stream=12345, size=10) # ===================================================================== @@ -808,12 +809,24 @@ async def test_async_stall_timeout_raises_transfer_stalled_error( ) # Simulate elapsed time 10.0 seconds during 10-byte upload (rate = 1 byte/s < 100) - clock_vals = iter([0.0, 10.0, 10.0, 10.0]) + clock_vals = iter([0.0, 10.0, 10.0]) monkeypatch.setattr(upload_async, "_monotonic_clock", lambda: next(clock_vals)) with pytest.raises(TransferStalledError, match="Upload stalled"): await session.upload(stream=b"0123456789") + # Verify branch where _stall_timeout_started is already set prior to chunk evaluation + clock_vals2 = iter([0.0, 10.0, 10.0]) + monkeypatch.setattr(upload_async, "_monotonic_clock", lambda: next(clock_vals2)) + session2 = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=config, + transport=DummyAsyncSession([start_resp, chunk_resp]), + ) + session2._stall_timeout_started = 0.0 + with pytest.raises(TransferStalledError, match="Upload stalled"): + await session2.upload(stream=b"0123456789") + @pytest.mark.asyncio async def test_async_deadline_exceeded() -> None: @@ -1085,67 +1098,55 @@ def test_async_deadline_handling_and_start_timeout() -> None: @pytest.mark.asyncio async def test_async_retry_branches() -> None: - """Verifies retry predicate branches in _async_retry.""" + """Verifies retry predicate and policy resolution branches in upload_async.""" session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", ) - # MissingStatusHeaderError retries and raises on final attempt - attempts = 0 + pred_start = session._get_retry_predicate(is_start=True) + pred_transfer = session._get_retry_predicate(is_start=False) - async def fail_missing_header(): - nonlocal attempts - attempts += 1 - raise exceptions.MissingStatusHeaderError("missing") + assert pred_start(exceptions.DeadlineExceeded("deadline")) is False + assert pred_start(exceptions.TransferStalledError("stalled")) is False + assert pred_start(exceptions.UploadCancelledError("cancelled")) is False + assert pred_start(exceptions.UnseekableStreamError("unseekable")) is False + assert pred_start(exceptions.MissingStatusHeaderError("missing")) is True + assert pred_start(aiohttp.ClientError("network error")) is True + assert pred_start(exceptions.from_http_status(503, "Service Unavailable")) is True + assert pred_start(exceptions.from_http_status(400, "Bad Request")) is False + assert pred_start(RuntimeError("runtime")) is False - with pytest.raises(exceptions.MissingStatusHeaderError): - await session._async_retry(fail_missing_header, max_attempts=2) - assert attempts == 2 + # Recoverable status code 500 is retryable during transfer + assert pred_transfer(exceptions.from_http_status(500, "Internal Error")) is True - # Non-retryable GoogleAPICallError raises immediately - async def fail_400(): - raise exceptions.from_http_status(400, "Bad Request") + # Default retry resolution + default_unary = session._get_async_retry(is_start=True) + assert isinstance(default_unary, google.api_core.retry.AsyncRetry) - with pytest.raises(exceptions.BadRequest): - await session._async_retry(fail_400, max_attempts=3) - - # Retryable GoogleAPICallError retries and raises on final attempt - attempts_503 = 0 - - async def fail_503(): - nonlocal attempts_503 - attempts_503 += 1 - raise exceptions.from_http_status(503, "Service Unavailable") - - with pytest.raises(exceptions.ServiceUnavailable): - await session._async_retry(fail_503, max_attempts=2) - assert attempts_503 == 2 - - # DeadlineExceeded propagates immediately (no retry) - async def fail_deadline(): - raise exceptions.DeadlineExceeded("deadline") - - with pytest.raises(exceptions.DeadlineExceeded): - await session._async_retry(fail_deadline, max_attempts=3) + default_stream = session._get_async_streaming_retry() + assert isinstance(default_stream, google.api_core.retry.AsyncStreamingRetry) - # General Exception propagates on last attempt - attempts_runtime = 0 - - async def fail_runtime(): - nonlocal attempts_runtime - attempts_runtime += 1 - raise RuntimeError("runtime") - - with pytest.raises(RuntimeError): - await session._async_retry(fail_runtime, max_attempts=2) - assert attempts_runtime == 2 - - # max_attempts=0 covers loop bypass falling through - async def dummy_async_func() -> int: - return 123 + # Custom AsyncRetry resolution + custom_unary = google.api_core.retry.AsyncRetry(initial=0.5) + session_unary = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(retry=custom_unary), + ) + assert session_unary._get_async_retry(is_start=True) is custom_unary + converted_stream = session_unary._get_async_streaming_retry() + assert isinstance(converted_stream, google.api_core.retry.AsyncStreamingRetry) + assert converted_stream._initial == 0.5 - res = await session._async_retry(dummy_async_func, max_attempts=0) - assert res is None + # Custom AsyncStreamingRetry resolution + custom_stream = google.api_core.retry.AsyncStreamingRetry(initial=0.25) + session_stream = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + config=ResumableUploadConfig(retry=custom_stream), + ) + assert session_stream._get_async_streaming_retry() is custom_stream + converted_unary = session_stream._get_async_retry(is_start=False) + assert isinstance(converted_unary, google.api_core.retry.AsyncRetry) + assert converted_unary._initial == 0.25 def test_async_transport_missing_errors() -> None: @@ -1426,8 +1427,7 @@ async def test_async_prepare_async_reader_additional_branches() -> None: ) # Inherit from concrete io.BytesIO so mypy recognizes these test streams as - # valid BinaryIO instances without requiring cast() or type: ignore (since - # typing.BinaryIO is an abstract class). + # valid BinaryIO instances natively. class LocalNoTellStream(io.BytesIO): """Simulates a stream that has read() but lacks getbuffer() and tell(). @@ -1470,6 +1470,12 @@ def tell(self) -> int: reader_fn3, size3, obj3 = session._prepare_async_reader(stream3, size=None) assert size3 is None + # 4. BinaryIO stream with explicit size (covers computed_size is not None branch) + reader_fn4, size4, obj4 = session._prepare_async_reader( + io.BytesIO(b"hello"), size=5 + ) + assert size4 == 5 + @pytest.mark.asyncio async def test_async_upload_empty_stream() -> None: @@ -1751,23 +1757,19 @@ def request( ) transport = StatefulAsyncTransport() - config = ResumableUploadConfig(chunk_size=4) + config = ResumableUploadConfig( + chunk_size=4, + retry=google.api_core.retry.AsyncStreamingRetry( + predicate=lambda exc: True, initial=0.001 + ), + ) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", config=config, transport=transport, ) - # Resumable uploads have two layers of retry: - # 1. HTTP-level retry (do_http): blindly re-sends the HTTP request on transient 503 errors. - # 2. Protocol-level recovery (_recover): triggered when HTTP retries exhaust; sends a "query" - # command to discover committed server offset and slices the active chunk buffer. - # Disable HTTP-level _async_retry here so the 503 immediately triggers protocol-level _recover(). - async def no_retry(coro_fn: Any, max_attempts: int = 1) -> Any: - return await coro_fn() - - with mock.patch.object(session, "_async_retry", side_effect=no_retry): - await session.upload(stream=b"012345") + await session.upload(stream=b"012345") # Verify no data loss occurred: server must receive all 6 bytes (b"012345"), not truncated b"0123" assert bytes(server_received_bytes) == b"012345" diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index f6262df3243d..8e93569a25e8 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -877,18 +877,23 @@ def test_sync_deadline_handling(): def test_sync_retry_predicate_branches(): - session = ResumableUploadSession(upload_url="https://api.example.com/init") - pred = session._get_retry_predicate() + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + ) + pred_transfer = session._get_retry_predicate(is_start=False) + pred_start = session._get_retry_predicate(is_start=True) - assert pred(exceptions.DeadlineExceeded("deadline")) is False - assert pred(TransferStalledError("stalled")) is False - assert pred(UploadCancelledError("cancelled")) is False - assert pred(MissingStatusHeaderError("missing")) is True - assert pred(requests.exceptions.ConnectionError("conn")) is True - assert pred(requests.exceptions.ChunkedEncodingError("chunked")) is True - assert pred(exceptions.from_http_status(503, "503")) is True - assert pred(exceptions.from_http_status(400, "400")) is False - assert pred(TypeError("other")) is False + assert pred_transfer(exceptions.DeadlineExceeded("deadline")) is False + assert pred_transfer(TransferStalledError("stalled")) is False + assert pred_transfer(UploadCancelledError("cancelled")) is False + assert pred_transfer(MissingStatusHeaderError("missing")) is True + assert pred_transfer(requests.exceptions.ConnectionError("conn")) is True + assert pred_transfer(requests.exceptions.ChunkedEncodingError("chunked")) is True + assert pred_transfer(exceptions.from_http_status(503, "503")) is True + assert pred_transfer(exceptions.from_http_status(400, "400")) is True + assert pred_start(exceptions.from_http_status(400, "400")) is False + assert pred_transfer(exceptions.from_http_status(403, "403")) is False + assert pred_transfer(TypeError("other")) is False def test_sync_reposition_stream_errors(): @@ -1031,14 +1036,14 @@ def test_sync_on_progress_and_capture(): config=config, ) session._state._resumable_url = "https://api.example.com/init" - with session._capture_progress(): - session._notify_progress(common.ProgressState.UPLOADING) - assert session._captured_progress is not None - assert len(session._captured_progress) == 1 - assert session._captured_progress[0].state == common.ProgressState.UPLOADING - assert callback_mock.called - assert callback_mock.call_args[0][0] is session._captured_progress[0] - assert session._captured_progress is None + progress_queue = [] + session._notify_progress( + common.ProgressState.UPLOADING, progress_queue=progress_queue + ) + assert len(progress_queue) == 1 + assert progress_queue[0].state == common.ProgressState.UPLOADING + assert callback_mock.called + assert callback_mock.call_args[0][0] is progress_queue[0] def test_sync_naive_deadline_tz(): @@ -1213,7 +1218,9 @@ def test_sync_transmit_chunk_timeout_with_stall_control_active(): transport=transport, ) session_dl._state._resumable_url = "https://upload.example.com/resumable-123" - session_dl._get_deadline_remaining = mock.Mock(side_effect=[5.0, -1.0]) + session_dl._get_deadline_remaining = mock.Mock( + side_effect=[5.0, exceptions.DeadlineExceeded("Deadline exceeded")] + ) with pytest.raises(exceptions.DeadlineExceeded): session_dl._transmit_chunk(transport, io.BytesIO(b"data"), size=4) @@ -1233,9 +1240,9 @@ def test_sync_transmit_chunk_timeout_outer_exception(): ) session._state._resumable_url = "https://upload.example.com/resumable-123" with pytest.raises(exceptions.TransferStalledError): - session._transmit_chunk(transport, io.BytesIO(b"data"), size=4) + list(session._transmit_all_chunks(transport, io.BytesIO(b"data"), 4)) - # To hit line 554-558 (outer exception handler with elapsed deadline) + # To hit outer exception handler in _transmit_all_chunks with elapsed deadline config_dl = ResumableUploadConfig( stall_minimum_rate=0, deadline=datetime.datetime.now(datetime.timezone.utc) @@ -1248,9 +1255,11 @@ def test_sync_transmit_chunk_timeout_outer_exception(): transport=transport, ) session_dl._state._resumable_url = "https://upload.example.com/resumable-123" - session_dl._get_deadline_remaining = mock.Mock(side_effect=[5.0, -1.0]) + session_dl._get_deadline_remaining = mock.Mock( + side_effect=[5.0, 5.0, exceptions.DeadlineExceeded("Deadline exceeded")] + ) with pytest.raises(exceptions.DeadlineExceeded): - session_dl._transmit_chunk(transport, io.BytesIO(b"data"), size=4) + list(session_dl._transmit_all_chunks(transport, io.BytesIO(b"data"), 4)) def test_sync_recover_failure(): @@ -1488,11 +1497,14 @@ def test_sync_transmit_all_chunks_captured_empty(): ) session._state._resumable_url = "https://upload.example.com/resumable-123" - session._captured_progress = None stream_obj = io.BytesIO(b"data") # Consume the generator - list(session._transmit_all_chunks(session_transport, stream_obj, 4)) + list( + session._transmit_all_chunks( + session_transport, stream_obj, 4, progress_queue=None + ) + ) def test_sync_upload_multiple_chunks(): @@ -1636,12 +1648,12 @@ def handle_request(method, url, data=None, headers=None, **kwargs): session_transport = mock.create_autospec(requests.Session, instance=True) session_transport.request.side_effect = handle_request - # Resumable uploads have two layers of retry: - # 1. HTTP-level retry (do_http): blindly re-sends the HTTP request on transient 503 errors. - # 2. Protocol-level recovery (_recover): triggered when HTTP retries exhaust; sends a "query" - # command to discover committed server offset and slices the active chunk buffer. - # Disable HTTP-level retry here so the 503 immediately triggers protocol-level _recover(). - retry_cfg = google.api_core.retry.Retry(predicate=lambda exc: False) + # Ensure that the unified outer StreamingRetry coordinates backoff and + # triggers protocol-level _recover() on retryable errors before retransmitting. + retry_cfg = google.api_core.retry.Retry( + predicate=ResumableUploadSession()._get_retry_predicate(), + initial=0.001, + ) config = ResumableUploadConfig(chunk_size=4, retry=retry_cfg) session = ResumableUploadSession( upload_url="https://api.example.com/init", @@ -1652,3 +1664,58 @@ def handle_request(method, url, data=None, headers=None, **kwargs): # Verify no data loss occurred: server must receive all 6 bytes (b"012345"), not truncated b"0123" assert bytes(server_received_bytes) == b"012345" + + +def test_sync_streaming_retry_and_recovery_final(): + """Ensure that StreamingRetry configuration and recovery returning 'final' status are handled properly.""" + session_transport = mock.create_autospec(requests.Session, instance=True) + + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-123", + }, + ) + err503_resp = mock.Mock( + ok=False, + status_code=503, + headers={}, + ) + err503_resp.json.return_value = {"error": {"code": 503, "message": "Unavailable"}} + err503_resp.text = '{"error": {"code": 503, "message": "Unavailable"}}' + err503_resp.content = err503_resp.text.encode("utf-8") + + # Recovery query reports that the upload already reached 'final' status on the server + query_final_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b'{"name": "already_done.txt", "size": 4}', + ) + + session_transport.request.side_effect = [ + start_resp, + err503_resp, + query_final_resp, + ] + + streaming_retry = google.api_core.retry.StreamingRetry( + predicate=ResumableUploadSession()._get_retry_predicate(), + initial=0.001, + ) + config = ResumableUploadConfig( + response_type=DummyResponse, + retry=streaming_retry, + ) + session = ResumableUploadSession( + upload_url="https://api.example.com/init", + config=config, + ) + assert session._get_streaming_retry() is streaming_retry + + result = session.upload(stream=b"data", transport=session_transport) + assert isinstance(result, DummyResponse) + assert result.name == "already_done.txt" + assert session.finished is True From 5f569623895791f426cd91108e5ffe51bf5c68b8 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 14:26:29 +0000 Subject: [PATCH 31/43] address review feedback --- .../resumable_transfer/upload_async.py | 24 ++++-- .../asyncio/test_resumable_transfer_async.py | 84 +++++++++++++++++++ 2 files changed, 100 insertions(+), 8 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 870b351f8b03..e1813c861176 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -91,6 +91,17 @@ def __init__( self._task = task self._session = session self._progress_queue = progress_queue + self._task.add_done_callback(self._on_task_done) + + def _on_task_done(self, task: asyncio.Task) -> None: + """Ensures progress queue unblocks when background task terminates.""" + if task.cancelled(): + self._progress_queue.put_nowait(asyncio.CancelledError()) + else: + exc = task.exception() + if exc is not None: + self._progress_queue.put_nowait(exc) + self._progress_queue.put_nowait(_DONE_SENTINEL) def __await__(self) -> Generator[Any, None, ResponseProto]: """Awaits completion of the upload task and returns the server response.""" @@ -103,13 +114,14 @@ async def progress(self) -> AsyncIterator[common.UploadProgress]: UploadProgress snapshots for each progress transition. Raises: - Exception: Re-raises any exception encountered during the background transfer. + BaseException: Re-raises any exception or cancellation encountered during the background transfer. """ while True: item = await self._progress_queue.get() if item is _DONE_SENTINEL: + self._progress_queue.put_nowait(_DONE_SENTINEL) break - if isinstance(item, Exception): + if isinstance(item, BaseException): raise item yield item @@ -799,11 +811,9 @@ async def _run(): _, _, body_bytes = final_resp_tuple self._response = self._format_response(body_bytes) - progress_queue.put_nowait(_DONE_SENTINEL) return self._response - except Exception as exc: + except BaseException as exc: self._enrich_exception(exc) - progress_queue.put_nowait(exc) raise task = asyncio.create_task(_run()) @@ -861,11 +871,9 @@ async def _run(): _, _, body_bytes = final_resp_tuple self._response = self._format_response(body_bytes) - progress_queue.put_nowait(_DONE_SENTINEL) return self._response - except Exception as exc: + except BaseException as exc: self._enrich_exception(exc) - progress_queue.put_nowait(exc) raise task = asyncio.create_task(_run()) diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index e690094cae68..95dc9e259955 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -1773,3 +1773,87 @@ def request( # Verify no data loss occurred: server must receive all 6 bytes (b"012345"), not truncated b"0123" assert bytes(server_received_bytes) == b"012345" + + +@pytest.mark.asyncio +async def test_async_upload_progress_cancellation_and_base_exception() -> None: + """Ensure progress() does not hang if the background task is cancelled or raises BaseException.""" + + class CustomBaseException(BaseException): + pass + + # 1. Task cancelled while awaiting progress() + slow_event = asyncio.Event() + + class HangingTransport: + def request(self, *args: Any, **kwargs: Any) -> Any: + class HangingCtx: + async def __aenter__(self) -> Any: + await slow_event.wait() + return DummyAsyncResponse(status=200, headers={}, body=b"") + + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + pass + + return HangingCtx() + + session_cancel = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=HangingTransport(), + ) + op_cancel = session_cancel.upload(stream=b"data") + + async def cancel_soon() -> None: + await asyncio.sleep(0.01) + op_cancel._task.cancel() + + cancel_task = asyncio.create_task(cancel_soon()) + with pytest.raises(asyncio.CancelledError): + async for _ in op_cancel.progress(): + pass + await cancel_task + + # 2. Task raises BaseException subclass + class BaseExceptionTransport: + def request(self, *args: Any, **kwargs: Any) -> Any: + class BaseExceptionCtx: + async def __aenter__(self) -> Any: + raise CustomBaseException("fatal error") + + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: + pass + + return BaseExceptionCtx() + + session_base_exc = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=BaseExceptionTransport(), + ) + op_base_exc = session_base_exc.upload(stream=b"data") + with pytest.raises(CustomBaseException, match="fatal error"): + async for _ in op_base_exc.progress(): + pass + + # 3. Multiple progress() iterations after completion do not hang + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/resumable-async", + }, + body=b"", + ) + final_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b"{}", + ) + session_ok = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=DummyAsyncSession([start_resp, final_resp]), + ) + op_ok = session_ok.upload(stream=b"data") + first_pass = [p async for p in op_ok.progress()] + assert len(first_pass) == 2 + second_pass = [p async for p in op_ok.progress()] + assert second_pass == [] From 73847dd1e7c46e3266b6914864303a64240863e2 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 14:36:08 +0000 Subject: [PATCH 32/43] fix build --- .../google/api_core/resumable_transfer/upload_async.py | 10 +++++----- .../tests/asyncio/test_resumable_transfer_async.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index e1813c861176..eafcd6e7c286 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -40,8 +40,10 @@ try: import aiohttp + + _HAS_AIOHTTP = True except ImportError: # pragma: NO COVER - pass + _HAS_AIOHTTP = False import google.api_core.retry from google.api_core import exceptions @@ -215,7 +217,7 @@ def _ensure_aiohttp(self) -> None: Raises: ImportError: If aiohttp is not installed. """ - if globals().get("aiohttp") is None: + if not _HAS_AIOHTTP: raise ImportError( "The aiohttp library is required to use AsyncResumableUploadSession. " "Please install google-api-core[async_rest]." @@ -321,9 +323,7 @@ def should_retry(exc: Any) -> bool: return False if isinstance(exc, exceptions.MissingStatusHeaderError): return True - if globals().get("aiohttp") is not None and isinstance( - exc, aiohttp.ClientError - ): + if _HAS_AIOHTTP and isinstance(exc, aiohttp.ClientError): return True if isinstance(exc, exceptions.GoogleAPICallError): return exc.code in allowed_codes diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 95dc9e259955..4d7185fdb2f4 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -217,7 +217,7 @@ async def test_async_cancel_missing_transport_raises() -> None: def test_async_ensure_aiohttp_missing(monkeypatch: pytest.MonkeyPatch) -> None: """Verifies that _ensure_aiohttp raises ImportError when aiohttp is unavailable.""" - monkeypatch.setattr(upload_async, "aiohttp", None) + monkeypatch.setattr(upload_async, "_HAS_AIOHTTP", False) session = AsyncResumableUploadSession() with pytest.raises(ImportError, match="google-api-core\\[async_rest\\]"): session._ensure_aiohttp() From 62fabec263a56a6d1f4f05757e9f02001bd0ab70 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 15:20:55 +0000 Subject: [PATCH 33/43] address review feedback --- .../api_core/resumable_transfer/common.py | 50 +---- .../api_core/resumable_transfer/upload.py | 196 +++++++++++++----- .../resumable_transfer/upload_async.py | 171 +++++++++++---- .../asyncio/test_resumable_transfer_async.py | 133 ++++++++---- .../tests/unit/test_resumable_transfer.py | 170 +++++++++------ 5 files changed, 472 insertions(+), 248 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/common.py b/packages/google-api-core/google/api_core/resumable_transfer/common.py index f426f5db26e8..9335f45cc911 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/common.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/common.py @@ -17,14 +17,12 @@ import dataclasses import datetime import enum -from typing import Any, Callable, Mapping, Optional, Sequence, Tuple, Union +from typing import Any, Mapping, Optional, Sequence, Tuple, Union import google.protobuf.message import proto from google.protobuf import json_format -import google.api_core.retry - # Default chunk size: 10 MiB DEFAULT_CHUNK_SIZE = 10 * 1024 * 1024 @@ -102,63 +100,17 @@ class ResumableUploadConfig: Attributes: chunk_size: Size in bytes for each uploaded data chunk. Defaults to 10 MiB. - start_timeout: Local per-request timeout in seconds for start request. - start_retry: Custom retry policy for the start request. stall_minimum_rate: Minimum transfer rate in bytes per second. Defaults to 64 KiB/s. stall_timeout: Stall duration threshold in seconds. Defaults to 120s. headers: Additional HTTP headers dispatched exclusively with start request. deadline: Overall global deadline for the upload process. - timeout: Fallback per-request timeout. - retry: Fallback retry policy. - on_progress: Callback function receiving UploadProgress notifications. - response_type: Optional message class (proto.Message or google.protobuf.message.Message), - callable deserializer, or None to return raw response. - content_type: MIME type of the stream payload. """ chunk_size: int = DEFAULT_CHUNK_SIZE - start_timeout: Optional[float] = None - start_retry: Optional[ - Union[google.api_core.retry.Retry, google.api_core.retry.AsyncRetry] - ] = None stall_minimum_rate: int = 64 * 1024 stall_timeout: float = 120.0 headers: Optional[Union[Mapping[str, str], Sequence[Tuple[str, str]]]] = None deadline: Optional[datetime.datetime] = None - timeout: Optional[float] = None - retry: Optional[ - Union[ - google.api_core.retry.Retry, - google.api_core.retry.StreamingRetry, - google.api_core.retry.AsyncRetry, - google.api_core.retry.AsyncStreamingRetry, - ] - ] = None - on_progress: Optional[Callable[[UploadProgress], None]] = None - response_type: Optional[Any] = None - content_type: Optional[str] = None - - def __post_init__(self) -> None: - """Normalizes fallback timeouts and retry policies.""" - if self.start_timeout is not None and self.timeout is None: - self.timeout = self.start_timeout - elif self.timeout is not None and self.start_timeout is None: - self.start_timeout = self.timeout - - if self.start_retry is not None and self.retry is None: - self.retry = self.start_retry - elif ( - self.start_retry is None - and self.retry is not None - and not isinstance( - self.retry, - ( - google.api_core.retry.StreamingRetry, - google.api_core.retry.AsyncStreamingRetry, - ), - ) - ): - self.start_retry = self.retry @property def start_headers(self) -> Optional[Sequence[Tuple[str, str]]]: diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index af9cfb7eae6f..29cfe482b954 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -101,6 +101,10 @@ def __init__( config: Optional[ResumableUploadConfig] = None, resumable_url: Optional[str] = None, transport: Optional[requests.Session] = None, + content_type: Optional[str] = None, + response_type: Optional[Any] = None, + start_retry: Optional[google.api_core.retry.Retry] = None, + start_timeout: Optional[float] = None, ) -> None: """Initializes a ResumableUploadSession. @@ -109,9 +113,18 @@ def __init__( config: Optional upload configuration parameters. resumable_url: Pre-existing upload session URL if resuming. transport: Optional requests session. + content_type: Optional MIME type of the stream payload. + response_type: Optional message class, callable deserializer, or None. + start_retry: Optional unary Retry policy for the start request. + start_timeout: Optional timeout in seconds for the start request. """ self._config = config or ResumableUploadConfig() self._transport = transport + self._content_type = content_type + self._response_type = response_type + self._start_retry = start_retry + self._start_timeout = start_timeout + self._on_progress: Optional[Callable[[common.UploadProgress], None]] = None self._response: Optional[Any] = None self._state = upload_state.ProtocolState( upload_url=upload_url, @@ -202,8 +215,8 @@ def _notify_progress( ) if progress_queue is not None: progress_queue.append(progress) - if self._config.on_progress: - self._config.on_progress(progress) + if self._on_progress: + self._on_progress(progress) def _get_deadline_remaining(self) -> Optional[float]: """Calculates remaining seconds until the configured upload deadline. @@ -227,15 +240,20 @@ def _get_deadline_remaining(self) -> Optional[float]: return remaining return None - def _get_start_timeout(self) -> float: + def _get_start_timeout(self, timeout_override: Optional[float] = None) -> float: """Computes timeout in seconds for start and control requests. + Args: + timeout_override: Explicit timeout override in seconds. + Returns: Applicable timeout in seconds. """ remaining = self._get_deadline_remaining() timeout = ( - self._config.start_timeout or self._config.timeout or _DEFAULT_START_TIMEOUT + timeout_override + if timeout_override is not None + else (self._start_timeout or _DEFAULT_START_TIMEOUT) ) if remaining is not None: return min(timeout, remaining) @@ -287,61 +305,61 @@ def should_retry(exc: Any) -> bool: return should_retry - def _get_retry(self, is_start: bool = False) -> google.api_core.retry.Retry: + def _get_retry( + self, retry_override: Optional[google.api_core.retry.Retry] = None + ) -> google.api_core.retry.Retry: """Resolves unary Retry policy for start requests. Args: - is_start: Whether this retry policy is for the start request. + retry_override: Optional unary Retry policy override for the start request. Returns: Configured or default unary Retry instance. """ - candidate = ( - self._config.start_retry - if is_start and self._config.start_retry - else self._config.retry - ) + candidate = retry_override or self._start_retry if candidate is not None: - if isinstance(candidate, google.api_core.retry.Retry): - return candidate - return google.api_core.retry.Retry( - predicate=candidate._predicate, - initial=candidate._initial, - maximum=candidate._maximum, - multiplier=candidate._multiplier, - timeout=candidate._timeout, - on_error=candidate._on_error, - ) + return candidate return google.api_core.retry.Retry( - predicate=self._get_retry_predicate(is_start=is_start) + predicate=self._get_retry_predicate(is_start=True) ) - def _get_streaming_retry(self) -> google.api_core.retry.StreamingRetry: + def _get_streaming_retry( + self, + retry_override: Optional[ + Union[google.api_core.retry.Retry, google.api_core.retry.StreamingRetry] + ] = None, + ) -> google.api_core.retry.StreamingRetry: """Resolves the StreamingRetry policy for the chunk upload generator. + Args: + retry_override: Optional retry policy override for chunk transmission. + Returns: Configured or default StreamingRetry instance. """ - if self._config.retry: - if isinstance(self._config.retry, google.api_core.retry.StreamingRetry): - return self._config.retry + if retry_override is not None: + if isinstance(retry_override, google.api_core.retry.StreamingRetry): + return retry_override return google.api_core.retry.StreamingRetry( - predicate=self._config.retry._predicate, - initial=self._config.retry._initial, - maximum=self._config.retry._maximum, - multiplier=self._config.retry._multiplier, - timeout=self._config.retry._timeout, - on_error=self._config.retry._on_error, + predicate=retry_override._predicate, + initial=retry_override._initial, + maximum=retry_override._maximum, + multiplier=retry_override._multiplier, + timeout=retry_override._timeout, + on_error=retry_override._on_error, ) return google.api_core.retry.StreamingRetry( predicate=self._get_retry_predicate(is_start=False) ) - def _compute_chunk_timeout(self, data_len: int) -> float: + def _compute_chunk_timeout( + self, data_len: int, timeout_override: Optional[float] = None + ) -> float: """Computes the dynamic per-attempt chunk timeout based on stall control and deadlines. Args: data_len: Length of the current chunk in bytes. + timeout_override: Optional per-attempt timeout ceiling in seconds. Returns: Timeout in seconds for chunk transmission attempt. @@ -354,8 +372,8 @@ def _compute_chunk_timeout(self, data_len: int) -> float: ) per_attempt_timeout = max(5.0, min(next_chunk_timeout, 2.0 * expected_sec)) - if self._config.timeout: - per_attempt_timeout = min(self._config.timeout, per_attempt_timeout) + if timeout_override is not None: + per_attempt_timeout = min(timeout_override, per_attempt_timeout) remaining = self._get_deadline_remaining() if remaining is not None: @@ -452,6 +470,9 @@ def initiate( request_body: Union[str, bytes] = "", size: Optional[int] = None, progress_queue: Optional[List[common.UploadProgress]] = None, + content_type: Optional[str] = None, + retry: Optional[google.api_core.retry.Retry] = None, + timeout: Optional[float] = None, ) -> str: """Initiates the upload session by sending the start command. @@ -460,21 +481,27 @@ def initiate( request_body: JSON payload for initial start request. size: Total size of payload in bytes, if known. progress_queue: Optional list buffering UploadProgress snapshots. + content_type: Optional MIME type override of the payload. + retry: Optional unary Retry policy override for the start request. + timeout: Optional per-request timeout override in seconds. Returns: The upload session URL. """ + if content_type is not None: + self._content_type = content_type + method, url, headers, payload = self._state.build_start_request( body=request_body, headers=self._config.start_headers, - content_type=self._config.content_type, + content_type=self._content_type, size=size, ) def do_initiate() -> str: - timeout = self._get_start_timeout() + req_timeout = self._get_start_timeout(timeout_override=timeout) response = transport.request( - method, url, data=payload, headers=headers, timeout=timeout + method, url, data=payload, headers=headers, timeout=req_timeout ) if not response.ok: raise exceptions.from_http_response(response) @@ -483,8 +510,8 @@ def do_initiate() -> str: ) return session_url - retry = self._get_retry(is_start=True) - retryable_initiate = retry(do_initiate) + retry_policy = self._get_retry(retry_override=retry) + retryable_initiate = retry_policy(do_initiate) session_url = retryable_initiate() self._notify_progress( common.ProgressState.STARTED, progress_queue=progress_queue @@ -497,6 +524,7 @@ def _transmit_chunk( stream: Union[BinaryIO, Iterable[bytes]], size: Optional[int], progress_queue: Optional[List[common.UploadProgress]] = None, + timeout: Optional[float] = None, ) -> requests.Response: """Transmits a single data chunk attempt with stall control. @@ -505,6 +533,7 @@ def _transmit_chunk( stream: The input data stream. size: Total size of the stream in bytes, if known. progress_queue: Optional list buffering UploadProgress snapshots. + timeout: Optional per-attempt timeout ceiling in seconds. Returns: The HTTP response for the transmitted chunk. @@ -537,10 +566,12 @@ def _transmit_chunk( method, url, headers, payload = self._state.build_chunk_request( data=data, is_last_chunk=is_last, - content_type=self._config.content_type, + content_type=self._content_type, ) - per_attempt_timeout = self._compute_chunk_timeout(data_len) + per_attempt_timeout = self._compute_chunk_timeout( + data_len, timeout_override=timeout + ) try: t_start = _monotonic_clock() resp = transport.request( @@ -636,6 +667,10 @@ def _transmit_all_chunks( stream_obj: Union[BinaryIO, Iterable[bytes]], computed_size: Optional[int], progress_queue: Optional[List[common.UploadProgress]] = None, + retry: Optional[ + Union[google.api_core.retry.Retry, google.api_core.retry.StreamingRetry] + ] = None, + timeout: Optional[float] = None, ) -> Generator[common.UploadProgress, None, None]: """Transmits chunks until transfer completes, yielding buffered progress updates. @@ -644,6 +679,8 @@ def _transmit_all_chunks( stream_obj: Binary stream yielding upload chunks. computed_size: Total payload size in bytes if known. progress_queue: Optional list buffering UploadProgress snapshots. + retry: Optional retry policy override for chunk transmission. + timeout: Optional per-attempt timeout ceiling in seconds. Yields: UploadProgress snapshots for each transmission milestone. @@ -685,6 +722,7 @@ def attempt_stream() -> Generator[common.UploadProgress, None, None]: stream_obj, computed_size, progress_queue=progress_queue, + timeout=timeout, ) except Exception as exc: is_recoverable = ( @@ -710,8 +748,8 @@ def attempt_stream() -> Generator[common.UploadProgress, None, None]: yield progress_queue.pop(0) try: - retry = self._get_streaming_retry() - retryable_stream = retry(attempt_stream) + retry_policy = self._get_streaming_retry(retry_override=retry) + retryable_stream = retry_policy(attempt_stream) yield from retryable_stream() except requests.exceptions.Timeout as exc: self._enrich_exception(exc) @@ -733,6 +771,12 @@ def upload( request_body: Union[str, bytes] = "", size: Optional[int] = None, transport: Optional[requests.Session] = None, + content_type: Optional[str] = None, + retry: Optional[ + Union[google.api_core.retry.Retry, google.api_core.retry.StreamingRetry] + ] = None, + timeout: Optional[float] = None, + on_progress: Optional[Callable[[common.UploadProgress], None]] = None, ) -> Any: """Executes the resumable upload from start to completion. @@ -741,6 +785,10 @@ def upload( request_body: Initial metadata payload sent with the start request. size: Total stream size in bytes, if known. transport: Optional requests session. + content_type: Optional MIME type of the stream payload. + retry: Optional retry policy override for chunk transmission. + timeout: Optional per-attempt timeout ceiling in seconds. + on_progress: Optional callback function receiving UploadProgress notifications. Returns: The final server response payload or deserialized response message. @@ -750,7 +798,14 @@ def upload( GoogleAPICallError: If an unrecoverable API error occurs. """ for _ in self.iter_upload( - stream=stream, request_body=request_body, size=size, transport=transport + stream=stream, + request_body=request_body, + size=size, + transport=transport, + content_type=content_type, + retry=retry, + timeout=timeout, + on_progress=on_progress, ): pass return self._response @@ -761,6 +816,12 @@ def iter_upload( request_body: Union[str, bytes] = "", size: Optional[int] = None, transport: Optional[requests.Session] = None, + content_type: Optional[str] = None, + retry: Optional[ + Union[google.api_core.retry.Retry, google.api_core.retry.StreamingRetry] + ] = None, + timeout: Optional[float] = None, + on_progress: Optional[Callable[[common.UploadProgress], None]] = None, ) -> Generator[common.UploadProgress, None, None]: """Streams upload execution, yielding UploadProgress snapshots (PEP 255). @@ -769,6 +830,10 @@ def iter_upload( request_body: Initial metadata payload sent with the start request. size: Total stream size in bytes, if known. transport: Optional requests session. + content_type: Optional MIME type of the stream payload. + retry: Optional retry policy override for chunk transmission. + timeout: Optional per-attempt timeout ceiling in seconds. + on_progress: Optional callback function receiving UploadProgress notifications. Yields: UploadProgress snapshots for each chunk transmission milestone. @@ -778,6 +843,10 @@ def iter_upload( GoogleAPICallError: If an unrecoverable API error occurs. """ sess = self._get_transport(transport) + if content_type is not None: + self._content_type = content_type + if on_progress is not None: + self._on_progress = on_progress progress_queue: List[common.UploadProgress] = [] try: stream_obj, computed_size = self._prepare_stream(stream, size) @@ -788,7 +857,12 @@ def iter_upload( progress_queue=progress_queue, ) yield from self._transmit_all_chunks( - sess, stream_obj, computed_size, progress_queue=progress_queue + sess, + stream_obj, + computed_size, + progress_queue=progress_queue, + retry=retry, + timeout=timeout, ) except Exception as exc: self._enrich_exception(exc) @@ -801,6 +875,11 @@ def resume( size: Optional[int] = None, chunk_size: Optional[int] = None, transport: Optional[requests.Session] = None, + retry: Optional[ + Union[google.api_core.retry.Retry, google.api_core.retry.StreamingRetry] + ] = None, + timeout: Optional[float] = None, + on_progress: Optional[Callable[[common.UploadProgress], None]] = None, ) -> Any: """Resumes an existing upload from a saved upload URL. @@ -810,6 +889,9 @@ def resume( size: Total size of the payload in bytes, if known. chunk_size: Optional chunk size override in bytes. transport: Optional requests session. + retry: Optional retry policy override for chunk transmission. + timeout: Optional per-attempt timeout ceiling in seconds. + on_progress: Optional callback function receiving UploadProgress notifications. Returns: The final server response payload or deserialized response message. @@ -824,6 +906,9 @@ def resume( size=size, chunk_size=chunk_size, transport=transport, + retry=retry, + timeout=timeout, + on_progress=on_progress, ): pass return self._response @@ -835,6 +920,11 @@ def iter_resume( size: Optional[int] = None, chunk_size: Optional[int] = None, transport: Optional[requests.Session] = None, + retry: Optional[ + Union[google.api_core.retry.Retry, google.api_core.retry.StreamingRetry] + ] = None, + timeout: Optional[float] = None, + on_progress: Optional[Callable[[common.UploadProgress], None]] = None, ) -> Generator[common.UploadProgress, None, None]: """Streams resumption of an upload, yielding UploadProgress snapshots. @@ -844,6 +934,9 @@ def iter_resume( size: Total size of the payload in bytes, if known. chunk_size: Optional chunk size override in bytes. transport: Optional requests session. + retry: Optional retry policy override for chunk transmission. + timeout: Optional per-attempt timeout ceiling in seconds. + on_progress: Optional callback function receiving UploadProgress notifications. Yields: UploadProgress snapshots for each chunk transmission milestone. @@ -861,6 +954,8 @@ def iter_resume( if chunk_size is not None: self._state._chunk_size = chunk_size + if on_progress is not None: + self._on_progress = on_progress self._state._resumable_url = actual_url progress_queue: List[common.UploadProgress] = [] @@ -868,7 +963,12 @@ def iter_resume( stream_obj, computed_size = self._prepare_stream(stream, size) self._recover(sess, stream_obj, progress_queue=progress_queue) yield from self._transmit_all_chunks( - sess, stream_obj, computed_size, progress_queue=progress_queue + sess, + stream_obj, + computed_size, + progress_queue=progress_queue, + retry=retry, + timeout=timeout, ) except Exception as exc: self._enrich_exception(exc) @@ -934,4 +1034,4 @@ def _format_response(self, response: requests.Response) -> Any: Returns: Deserialized protobuf message or the raw response object. """ - return _format_response_payload(response, self._config.response_type) + return _format_response_payload(response, self._response_type) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index eafcd6e7c286..06a270429646 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -157,6 +157,10 @@ def __init__( config: Optional[ResumableUploadConfig] = None, resumable_url: Optional[str] = None, transport: Optional[Any] = None, + content_type: Optional[str] = None, + response_type: Optional[Any] = None, + start_retry: Optional[google.api_core.retry.AsyncRetry] = None, + start_timeout: Optional[float] = None, ) -> None: """Initializes an AsyncResumableUploadSession. @@ -165,9 +169,18 @@ def __init__( config: Optional upload configuration parameters. resumable_url: Pre-existing upload session URL if resuming. transport: Optional aiohttp.ClientSession. + content_type: Optional MIME type of the stream payload. + response_type: Optional message class, callable deserializer, or None. + start_retry: Optional unary AsyncRetry policy for the start request. + start_timeout: Optional timeout in seconds for the start request. """ self._config = config or ResumableUploadConfig() self._transport = transport + self._content_type = content_type + self._response_type = response_type + self._start_retry = start_retry + self._start_timeout = start_timeout + self._on_progress: Optional[Callable[[common.UploadProgress], Any]] = None self._response: Optional[Any] = None self._state = upload_state.ProtocolState( upload_url=upload_url, @@ -249,9 +262,9 @@ def _notify_progress( total_bytes=self._state.total_bytes, state=state, ) - if self._config.on_progress: + if self._on_progress: try: - self._config.on_progress(progress) + self._on_progress(progress) except Exception: # pragma: NO COVER pass if queue is not None: @@ -279,15 +292,20 @@ def _get_deadline_remaining(self) -> Optional[float]: return remaining return None - def _get_start_timeout(self) -> float: + def _get_start_timeout(self, timeout_override: Optional[float] = None) -> float: """Computes timeout in seconds for start and control requests. + Args: + timeout_override: Explicit timeout override in seconds. + Returns: Applicable timeout in seconds. """ remaining = self._get_deadline_remaining() timeout = ( - self._config.start_timeout or self._config.timeout or _DEFAULT_START_TIMEOUT + timeout_override + if timeout_override is not None + else (self._start_timeout or _DEFAULT_START_TIMEOUT) ) if remaining is not None: return min(timeout, remaining) @@ -332,56 +350,50 @@ def should_retry(exc: Any) -> bool: return should_retry def _get_async_retry( - self, is_start: bool = False + self, retry_override: Optional[google.api_core.retry.AsyncRetry] = None ) -> google.api_core.retry.AsyncRetry: """Resolves unary AsyncRetry policy for start requests. Args: - is_start: Whether this retry policy is for the start request. + retry_override: Optional unary AsyncRetry policy override for the start request. Returns: Configured or default unary AsyncRetry instance. """ - candidate = ( - self._config.start_retry - if is_start and self._config.start_retry - else self._config.retry - ) + candidate = retry_override or self._start_retry if candidate is not None: - if isinstance(candidate, google.api_core.retry.AsyncRetry): - return candidate - return google.api_core.retry.AsyncRetry( - predicate=candidate._predicate, - initial=candidate._initial, - maximum=candidate._maximum, - multiplier=candidate._multiplier, - timeout=candidate._timeout, - on_error=candidate._on_error, - ) + return candidate return google.api_core.retry.AsyncRetry( - predicate=self._get_retry_predicate(is_start=is_start) + predicate=self._get_retry_predicate(is_start=True) ) def _get_async_streaming_retry( self, + retry_override: Optional[ + Union[ + google.api_core.retry.AsyncRetry, + google.api_core.retry.AsyncStreamingRetry, + ] + ] = None, ) -> google.api_core.retry.AsyncStreamingRetry: """Resolves the AsyncStreamingRetry policy for the chunk upload generator. + Args: + retry_override: Optional retry policy override for chunk transmission. + Returns: Configured or default AsyncStreamingRetry instance. """ - if self._config.retry: - if isinstance( - self._config.retry, google.api_core.retry.AsyncStreamingRetry - ): - return self._config.retry + if retry_override is not None: + if isinstance(retry_override, google.api_core.retry.AsyncStreamingRetry): + return retry_override return google.api_core.retry.AsyncStreamingRetry( - predicate=self._config.retry._predicate, - initial=self._config.retry._initial, - maximum=self._config.retry._maximum, - multiplier=self._config.retry._multiplier, - timeout=self._config.retry._timeout, - on_error=self._config.retry._on_error, + predicate=retry_override._predicate, + initial=retry_override._initial, + maximum=retry_override._maximum, + multiplier=retry_override._multiplier, + timeout=retry_override._timeout, + on_error=retry_override._on_error, ) return google.api_core.retry.AsyncStreamingRetry( predicate=self._get_retry_predicate(is_start=False) @@ -393,6 +405,9 @@ async def initiate( request_body: Union[str, bytes] = "", size: Optional[int] = None, progress_queue: Optional[asyncio.Queue] = None, + content_type: Optional[str] = None, + retry: Optional[google.api_core.retry.AsyncRetry] = None, + timeout: Optional[float] = None, ) -> str: """Initiates the upload session by sending the start command asynchronously. @@ -401,6 +416,9 @@ async def initiate( request_body: Initial metadata payload sent with start command. size: Total stream size in bytes, if known. progress_queue: Optional queue to receive progress event. + content_type: Optional MIME type override of the payload. + retry: Optional unary AsyncRetry policy override for the start request. + timeout: Optional per-request timeout override in seconds. Returns: The upload session URL. @@ -410,15 +428,18 @@ async def initiate( MissingStatusHeaderError: If the server response lacks status header. """ self._ensure_aiohttp() + if content_type is not None: + self._content_type = content_type + method, url, headers, payload = self._state.build_start_request( body=request_body, headers=self._config.start_headers, - content_type=self._config.content_type, + content_type=self._content_type, size=size, ) async def do_initiate(): - timeout_sec = self._get_start_timeout() + timeout_sec = self._get_start_timeout(timeout_override=timeout) client_timeout = aiohttp.ClientTimeout(total=timeout_sec) async with transport.request( method, url, data=payload, headers=headers, timeout=client_timeout @@ -434,8 +455,8 @@ async def do_initiate(): ) return session_url - retry = self._get_async_retry(is_start=True) - retryable_initiate = retry(do_initiate) + retry_policy = self._get_async_retry(retry_override=retry) + retryable_initiate = retry_policy(do_initiate) session_url = await retryable_initiate() self._notify_progress(common.ProgressState.STARTED, progress_queue) return session_url @@ -446,6 +467,7 @@ async def _transmit_chunk( reader_fn: Callable[[int], Awaitable[bytes]], size: Optional[int], progress_queue: Optional[asyncio.Queue] = None, + timeout: Optional[float] = None, ) -> Tuple[int, Mapping[str, str], bytes]: """Transmits the next data chunk asynchronously with stall control. @@ -454,6 +476,7 @@ async def _transmit_chunk( reader_fn: Async callable returning chunk bytes. size: Total stream size in bytes, if known. progress_queue: Optional queue to receive progress updates. + timeout: Optional per-attempt timeout ceiling in seconds. Returns: Tuple of (status code, headers mapping, response body bytes). @@ -490,7 +513,7 @@ async def _transmit_chunk( method, url, headers, payload = self._state.build_chunk_request( data=data, is_last_chunk=is_last, - content_type=self._config.content_type, + content_type=self._content_type, ) rate = self._config.stall_minimum_rate @@ -501,8 +524,8 @@ async def _transmit_chunk( ) per_attempt_timeout = max(5.0, min(next_chunk_timeout, 2.0 * expected_sec)) - if self._config.timeout: - per_attempt_timeout = min(self._config.timeout, per_attempt_timeout) + if timeout is not None: + per_attempt_timeout = min(timeout, per_attempt_timeout) remaining = self._get_deadline_remaining() if remaining is not None: @@ -687,6 +710,13 @@ async def _transmit_all_chunks( computed_size: Optional[int], progress_queue: Optional[asyncio.Queue] = None, stream_obj: Optional[object] = None, + retry: Optional[ + Union[ + google.api_core.retry.AsyncRetry, + google.api_core.retry.AsyncStreamingRetry, + ] + ] = None, + timeout: Optional[float] = None, ) -> Optional[Tuple[int, Mapping[str, str], bytes]]: """Transmits chunks until completion using a single outer AsyncStreamingRetry coordinator. @@ -696,6 +726,8 @@ async def _transmit_all_chunks( computed_size: Total stream size in bytes, if known. progress_queue: Optional queue receiving UploadProgress snapshots. stream_obj: Underlying stream object for recovery seeking. + retry: Optional retry policy override for chunk transmission. + timeout: Optional per-attempt timeout ceiling in seconds. Returns: Tuple of (status code, headers, body bytes) of the final server response, or None. @@ -720,7 +752,11 @@ async def attempt_stream() -> AsyncGenerator[None, None]: while not self._state.finished and not self._state.invalid: try: final_resp_tuple = await self._transmit_chunk( - transport, reader_fn, computed_size, progress_queue + transport, + reader_fn, + computed_size, + progress_queue, + timeout=timeout, ) except Exception as exc: is_recoverable = ( @@ -743,8 +779,8 @@ async def attempt_stream() -> AsyncGenerator[None, None]: yield try: - retry = self._get_async_streaming_retry() - retryable_stream = retry(attempt_stream) + retry_policy = self._get_async_streaming_retry(retry_override=retry) + retryable_stream = retry_policy(attempt_stream) stream_gen = await retryable_stream() async for _ in stream_gen: pass @@ -768,6 +804,15 @@ def upload( request_body: Union[str, bytes] = "", size: Optional[int] = None, transport: Optional[Any] = None, + content_type: Optional[str] = None, + retry: Optional[ + Union[ + google.api_core.retry.AsyncRetry, + google.api_core.retry.AsyncStreamingRetry, + ] + ] = None, + timeout: Optional[float] = None, + on_progress: Optional[Callable[[common.UploadProgress], Any]] = None, ) -> AsyncUploadOperation: """Initiates and executes upload asynchronously, returning an AsyncUploadOperation. @@ -776,6 +821,10 @@ def upload( request_body: Initial metadata payload sent with the start request. size: Total stream size in bytes, if known. transport: Optional aiohttp client session. + content_type: Optional MIME type of the stream payload. + retry: Optional retry policy override for chunk transmission. + timeout: Optional per-attempt timeout ceiling in seconds. + on_progress: Optional callback function receiving UploadProgress notifications. Returns: An AsyncUploadOperation handle representing the active transfer. @@ -788,6 +837,11 @@ def upload( if sess is None: raise ValueError("An aiohttp.ClientSession transport must be provided.") + if content_type is not None: + self._content_type = content_type + if on_progress is not None: + self._on_progress = on_progress + progress_queue: asyncio.Queue = asyncio.Queue() reader_fn, computed_size, stream_obj = self._prepare_async_reader(stream, size) @@ -801,7 +855,13 @@ async def _run(): ) final_resp_tuple = await self._transmit_all_chunks( - sess, reader_fn, computed_size, progress_queue, stream_obj + sess, + reader_fn, + computed_size, + progress_queue, + stream_obj, + retry=retry, + timeout=timeout, ) if final_resp_tuple is None: @@ -828,6 +888,14 @@ def resume( size: Optional[int] = None, chunk_size: Optional[int] = None, transport: Optional[Any] = None, + retry: Optional[ + Union[ + google.api_core.retry.AsyncRetry, + google.api_core.retry.AsyncStreamingRetry, + ] + ] = None, + timeout: Optional[float] = None, + on_progress: Optional[Callable[[common.UploadProgress], Any]] = None, ) -> AsyncUploadOperation: """Resumes an existing upload asynchronously, returning an AsyncUploadOperation. @@ -837,6 +905,9 @@ def resume( size: Total stream size in bytes, if known. chunk_size: Optional chunk size override in bytes. transport: Optional aiohttp client session. + retry: Optional retry policy override for chunk transmission. + timeout: Optional per-attempt timeout ceiling in seconds. + on_progress: Optional callback function receiving UploadProgress notifications. Returns: An AsyncUploadOperation handle representing the resumed transfer. @@ -851,6 +922,8 @@ def resume( if chunk_size is not None: self._state._chunk_size = chunk_size + if on_progress is not None: + self._on_progress = on_progress self._state._resumable_url = upload_url progress_queue: asyncio.Queue = asyncio.Queue() @@ -861,7 +934,13 @@ async def _run(): await self._recover(sess, stream_obj, progress_queue=progress_queue) final_resp_tuple = await self._transmit_all_chunks( - sess, reader_fn, computed_size, progress_queue, stream_obj + sess, + reader_fn, + computed_size, + progress_queue, + stream_obj, + retry=retry, + timeout=timeout, ) if final_resp_tuple is None: @@ -995,4 +1074,4 @@ def _format_response(self, response_bytes: bytes) -> Any: Returns: Deserialized protobuf message or the raw bytes response. """ - return _format_response_payload(response_bytes, self._config.response_type) + return _format_response_payload(response_bytes, self._response_type) diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 4d7185fdb2f4..0f2a6043c844 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -246,10 +246,9 @@ async def test_async_upload_direct_execution() -> None: ) async_transport = DummyAsyncSession([start_resp, chunk_resp]) - config = ResumableUploadConfig(response_type=DummyResponse) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=config, + response_type=DummyResponse, transport=async_transport, ) @@ -286,10 +285,11 @@ async def test_async_upload_multi_chunk_operation_handle() -> None: ) async_transport = DummyAsyncSession([start_resp, chunk1_resp, chunk2_resp]) - config = ResumableUploadConfig(chunk_size=4, response_type=DummyResponse) + config = ResumableUploadConfig(chunk_size=4) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", config=config, + response_type=DummyResponse, transport=async_transport, ) @@ -344,10 +344,11 @@ async def test_async_upload_progress_tracking() -> None: ) async_transport = DummyAsyncSession([start_resp, chunk1_resp, chunk2_resp]) - config = ResumableUploadConfig(chunk_size=4, response_type=DummyResponse) + config = ResumableUploadConfig(chunk_size=4) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", config=config, + response_type=DummyResponse, transport=async_transport, ) @@ -396,7 +397,7 @@ async def async_generator() -> AsyncIterator[bytes]: session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=async_transport, ) @@ -425,7 +426,7 @@ async def test_async_stream_types_binary_io() -> None: async_transport = DummyAsyncSession([start_resp, chunk_resp]) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=async_transport, ) @@ -455,7 +456,7 @@ async def test_async_stream_types_sync_iterable() -> None: async_transport = DummyAsyncSession([start_resp, chunk_resp]) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=async_transport, ) @@ -496,7 +497,7 @@ async def test_async_resume_success() -> None: async_transport = DummyAsyncSession([query_resp, chunk_resp]) session = AsyncResumableUploadSession( - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=async_transport, ) @@ -529,7 +530,7 @@ async def test_async_resume_recovery_unseekable_stream_raises() -> None: async_transport = DummyAsyncSession([query_resp]) session = AsyncResumableUploadSession( - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=async_transport, ) @@ -564,7 +565,7 @@ async def test_async_resume_recovery_seekable_stream() -> None: async_transport = DummyAsyncSession([query_resp, chunk_resp]) session = AsyncResumableUploadSession( - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=async_transport, ) @@ -653,7 +654,7 @@ async def test_async_retry_transient_http_errors() -> None: async_transport = DummyAsyncSession([start_resp, chunk_503, chunk_success]) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=async_transport, ) @@ -721,7 +722,7 @@ async def test_async_recoverable_status_code_triggers_recovery() -> None: ) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=async_transport, ) @@ -765,7 +766,7 @@ async def test_async_missing_status_header_triggers_recovery() -> None: ) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=async_transport, ) @@ -879,7 +880,7 @@ async def test_async_response_type_proto_message() -> None: async_transport = DummyAsyncSession([start_resp, chunk_resp]) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=EchoResponse), + response_type=EchoResponse, transport=async_transport, ) @@ -908,7 +909,7 @@ async def test_async_response_type_protobuf_message() -> None: async_transport = DummyAsyncSession([start_resp, chunk_resp]) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=empty_pb2.Empty), + response_type=empty_pb2.Empty, transport=async_transport, ) @@ -943,7 +944,7 @@ def custom_parser(raw: bytes) -> str: session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=custom_parser), + response_type=custom_parser, transport=async_transport, ) @@ -1050,11 +1051,10 @@ def test_async_enrich_exception() -> None: def test_async_notify_progress_branches() -> None: """Verifies progress notification callbacks and queues.""" called = [] - config = ResumableUploadConfig(on_progress=lambda p: called.append(p)) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=config, ) + session._on_progress = lambda p: called.append(p) # When upload_url is None session._notify_progress(common.ProgressState.UPLOADING) assert len(called) == 0 @@ -1120,33 +1120,31 @@ async def test_async_retry_branches() -> None: assert pred_transfer(exceptions.from_http_status(500, "Internal Error")) is True # Default retry resolution - default_unary = session._get_async_retry(is_start=True) + default_unary = session._get_async_retry() assert isinstance(default_unary, google.api_core.retry.AsyncRetry) default_stream = session._get_async_streaming_retry() assert isinstance(default_stream, google.api_core.retry.AsyncStreamingRetry) - # Custom AsyncRetry resolution + # Custom AsyncRetry resolution via start_retry and retry_override custom_unary = google.api_core.retry.AsyncRetry(initial=0.5) session_unary = AsyncResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(retry=custom_unary), + start_retry=custom_unary, + ) + assert session_unary._get_async_retry() is custom_unary + converted_stream = session_unary._get_async_streaming_retry( + retry_override=custom_unary ) - assert session_unary._get_async_retry(is_start=True) is custom_unary - converted_stream = session_unary._get_async_streaming_retry() assert isinstance(converted_stream, google.api_core.retry.AsyncStreamingRetry) assert converted_stream._initial == 0.5 - # Custom AsyncStreamingRetry resolution + # Custom AsyncStreamingRetry resolution via retry_override custom_stream = google.api_core.retry.AsyncStreamingRetry(initial=0.25) - session_stream = AsyncResumableUploadSession( - upload_url="https://api.example.com/start", - config=ResumableUploadConfig(retry=custom_stream), + assert ( + session._get_async_streaming_retry(retry_override=custom_stream) + is custom_stream ) - assert session_stream._get_async_streaming_retry() is custom_stream - converted_unary = session_stream._get_async_retry(is_start=False) - assert isinstance(converted_unary, google.api_core.retry.AsyncRetry) - assert converted_unary._initial == 0.25 def test_async_transport_missing_errors() -> None: @@ -1281,7 +1279,7 @@ async def test_async_upload_with_timeout_and_deadline() -> None: future_deadline = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( seconds=60 ) - config = ResumableUploadConfig(timeout=30.0, deadline=future_deadline) + config = ResumableUploadConfig(deadline=future_deadline) start_resp = DummyAsyncResponse( status=200, @@ -1304,7 +1302,7 @@ async def test_async_upload_with_timeout_and_deadline() -> None: session._state._resumable_url = "https://upload.example.com/resumable-async" # Run the upload - res = await session.upload(stream=b"data") + res = await session.upload(stream=b"data", timeout=30.0) assert res == b"{}" @@ -1759,9 +1757,6 @@ def request( transport = StatefulAsyncTransport() config = ResumableUploadConfig( chunk_size=4, - retry=google.api_core.retry.AsyncStreamingRetry( - predicate=lambda exc: True, initial=0.001 - ), ) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", @@ -1769,7 +1764,12 @@ def request( transport=transport, ) - await session.upload(stream=b"012345") + await session.upload( + stream=b"012345", + retry=google.api_core.retry.AsyncStreamingRetry( + predicate=lambda exc: True, initial=0.001 + ), + ) # Verify no data loss occurred: server must receive all 6 bytes (b"012345"), not truncated b"0123" assert bytes(server_received_bytes) == b"012345" @@ -1857,3 +1857,62 @@ async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None: assert len(first_pass) == 2 second_pass = [p async for p in op_ok.progress()] assert second_pass == [] + + +@pytest.mark.asyncio +async def test_async_method_override_arguments() -> None: + """Verifies content_type and on_progress overrides on initiate, upload, and resume.""" + start_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/123", + }, + body=b"", + ) + chunk_resp = DummyAsyncResponse( + status=200, + headers={"X-Goog-Upload-Status": "final"}, + body=b"{}", + ) + + # 1. initiate with content_type override + session1 = AsyncResumableUploadSession(upload_url="https://api.example.com/start") + await session1.initiate( + transport=DummyAsyncSession([start_resp]), content_type="text/plain" + ) + assert session1._content_type == "text/plain" + + # 2. upload with content_type and on_progress overrides + progress_events: List[UploadProgress] = [] + session2 = AsyncResumableUploadSession( + upload_url="https://api.example.com/start", + transport=DummyAsyncSession([start_resp, chunk_resp]), + ) + await session2.upload( + stream=b"data", + content_type="text/csv", + on_progress=lambda p: progress_events.append(p), + ) + assert session2._content_type == "text/csv" + assert len(progress_events) == 2 + + # 3. resume with on_progress override + query_resp = DummyAsyncResponse( + status=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "0", + }, + body=b"", + ) + resume_events: List[UploadProgress] = [] + session3 = AsyncResumableUploadSession( + transport=DummyAsyncSession([query_resp, chunk_resp]) + ) + await session3.resume( + upload_url="https://upload.example.com/123", + stream=b"data", + on_progress=lambda p: resume_events.append(p), + ) + assert len(resume_events) == 2 diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 8e93569a25e8..019b86620dd5 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -14,7 +14,7 @@ import datetime import io -from typing import Union +from typing import List, Union from unittest import mock import pytest @@ -270,35 +270,19 @@ def test_resumable_upload_config_defaults(): assert config.chunk_size == 10 * 1024 * 1024 assert config.stall_minimum_rate == 64 * 1024 assert config.stall_timeout == 120.0 - assert config.start_timeout is None - assert config.start_retry is None assert config.headers is None assert config.deadline is None -def test_resumable_upload_config_fallbacks_and_headers(): - retry1 = mock.Mock() +def test_resumable_upload_config_headers(): config1 = ResumableUploadConfig( - start_timeout=45.0, - retry=retry1, headers={"X-Test": "1"}, ) - assert config1.timeout == 45.0 - assert config1.start_timeout == 45.0 - assert config1.retry is retry1 - assert config1.start_retry is retry1 assert config1.start_headers == [("X-Test", "1")] - retry2 = mock.Mock() config2 = ResumableUploadConfig( - timeout=30.0, - start_retry=retry2, headers=[("X-Test", "2")], ) - assert config2.timeout == 30.0 - assert config2.start_timeout == 30.0 - assert config2.retry is retry2 - assert config2.start_retry is retry2 assert config2.start_headers == [("X-Test", "2")] @@ -328,11 +312,10 @@ def test_sync_upload_direct_execution(): session_transport.request.side_effect = [start_resp, chunk_resp] - config = ResumableUploadConfig(response_type=DummyResponse) session = ResumableUploadSession( upload_url="https://api.example.com/start", - config=config, transport=session_transport, + response_type=DummyResponse, ) payload = b"Hello world" @@ -372,11 +355,12 @@ def test_sync_upload_iterative_progress(): session_transport.request.side_effect = [start_resp, chunk1_resp, chunk2_resp] - config = ResumableUploadConfig(chunk_size=4, response_type=DummyResponse) + config = ResumableUploadConfig(chunk_size=4) session = ResumableUploadSession( upload_url="https://api.example.com/start", config=config, transport=session_transport, + response_type=DummyResponse, ) progress_events = list(session.iter_upload(stream=b"12345678")) @@ -412,8 +396,7 @@ def test_sync_resume(): ) session_transport.request.side_effect = [query_resp, chunk_resp] - config = ResumableUploadConfig(response_type=DummyResponse) - session = ResumableUploadSession(config=config) + session = ResumableUploadSession(response_type=DummyResponse) stream = io.BytesIO(b"0123456789") resp = session.resume( @@ -448,8 +431,7 @@ def test_sync_iter_resume(): ) session_transport.request.side_effect = [query_resp, chunk_resp] - config = ResumableUploadConfig(response_type=DummyResponse) - session = ResumableUploadSession(config=config) + session = ResumableUploadSession(response_type=DummyResponse) stream = io.BytesIO(b"0123456789") progress_list = list( @@ -514,7 +496,7 @@ def test_sync_recoverable_status_code_triggers_offset_recovery(): session = ResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=DummyResponse), + response_type=DummyResponse, transport=session_transport, ) @@ -656,7 +638,7 @@ def test_sync_response_type_proto_message(): session = ResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=EchoResponse), + response_type=EchoResponse, transport=session_transport, ) resp = session.upload(stream=b"payload") @@ -684,7 +666,7 @@ def test_sync_response_type_protobuf_message(): session = ResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=empty_pb2.Empty), + response_type=empty_pb2.Empty, transport=session_transport, ) resp = session.upload(stream=b"payload") @@ -711,7 +693,7 @@ def test_sync_response_type_callable(): session = ResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=lambda c: c.decode("utf-8").upper()), + response_type=lambda c: c.decode("utf-8").upper(), transport=session_transport, ) resp = session.upload(stream=b"payload") @@ -738,7 +720,7 @@ def test_sync_response_type_raw_response(): session = ResumableUploadSession( upload_url="https://api.example.com/start", - config=ResumableUploadConfig(response_type=None), + response_type=None, transport=session_transport, ) resp = session.upload(stream=b"payload") @@ -956,20 +938,7 @@ def seekable(self) -> bool: assert custom.tell() == 2 -def test_sync_config_fallbacks_and_headers(): - cfg1 = ResumableUploadConfig(start_timeout=15.0) - assert cfg1.timeout == 15.0 - - cfg2 = ResumableUploadConfig(timeout=25.0) - assert cfg2.start_timeout == 25.0 - - ret = mock.Mock(spec=google.api_core.retry.Retry) - cfg3 = ResumableUploadConfig(start_retry=ret) - assert cfg3.retry is ret - - cfg4 = ResumableUploadConfig(retry=ret) - assert cfg4.start_retry is ret - +def test_sync_config_headers(): cfg_dict = ResumableUploadConfig(headers={"X-Key": "Val"}) assert cfg_dict.start_headers == [("X-Key", "Val")] @@ -1030,11 +999,10 @@ def test_sync_resume_chunk_size_override(): def test_sync_on_progress_and_capture(): callback_mock = mock.Mock() - config = ResumableUploadConfig(on_progress=callback_mock) session = ResumableUploadSession( upload_url="https://api.example.com/init", - config=config, ) + session._on_progress = callback_mock session._state._resumable_url = "https://api.example.com/init" progress_queue = [] session._notify_progress( @@ -1058,32 +1026,30 @@ def test_sync_naive_deadline_tz(): assert session._get_start_timeout() <= rem -def test_sync_get_retry_start_and_fallback(): +def test_sync_get_retry_start_and_override(): ret_start = mock.Mock(spec=google.api_core.retry.Retry) - ret_fallback = mock.Mock(spec=google.api_core.retry.Retry) - config = ResumableUploadConfig(start_retry=ret_start, retry=ret_fallback) + ret_override = mock.Mock(spec=google.api_core.retry.Retry) session = ResumableUploadSession( upload_url="https://api.example.com/init", - config=config, + start_retry=ret_start, ) - assert session._get_retry(is_start=True) is ret_start - assert session._get_retry() is ret_fallback + assert session._get_retry() is ret_start + assert session._get_retry(retry_override=ret_override) is ret_override def test_sync_stall_control_with_deadline(): import time - # 1. compute_chunk_timeout with fallback timeout and stall control active + # 1. compute_chunk_timeout with timeout_override and stall control active config = ResumableUploadConfig( stall_minimum_rate=1024, stall_timeout=10.0, - timeout=15.0, ) session = ResumableUploadSession( upload_url="https://api.example.com/init", config=config, ) - t1 = session._compute_chunk_timeout(512) + t1 = session._compute_chunk_timeout(512, timeout_override=15.0) assert t1 <= 15.0 # 2. compute_chunk_timeout with deadline active @@ -1231,7 +1197,6 @@ def test_sync_transmit_chunk_timeout_outer_exception(): config = ResumableUploadConfig( stall_minimum_rate=0, - retry=google.api_core.retry.Retry(predicate=lambda e: False), ) session = ResumableUploadSession( upload_url="https://api.example.com/init", @@ -1239,15 +1204,19 @@ def test_sync_transmit_chunk_timeout_outer_exception(): transport=transport, ) session._state._resumable_url = "https://upload.example.com/resumable-123" + no_retry = google.api_core.retry.Retry(predicate=lambda e: False) with pytest.raises(exceptions.TransferStalledError): - list(session._transmit_all_chunks(transport, io.BytesIO(b"data"), 4)) + list( + session._transmit_all_chunks( + transport, io.BytesIO(b"data"), 4, retry=no_retry + ) + ) # To hit outer exception handler in _transmit_all_chunks with elapsed deadline config_dl = ResumableUploadConfig( stall_minimum_rate=0, deadline=datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=5), - retry=google.api_core.retry.Retry(predicate=lambda e: False), ) session_dl = ResumableUploadSession( upload_url="https://api.example.com/init", @@ -1259,7 +1228,11 @@ def test_sync_transmit_chunk_timeout_outer_exception(): side_effect=[5.0, 5.0, exceptions.DeadlineExceeded("Deadline exceeded")] ) with pytest.raises(exceptions.DeadlineExceeded): - list(session_dl._transmit_all_chunks(transport, io.BytesIO(b"data"), 4)) + list( + session_dl._transmit_all_chunks( + transport, io.BytesIO(b"data"), 4, retry=no_retry + ) + ) def test_sync_recover_failure(): @@ -1654,13 +1627,13 @@ def handle_request(method, url, data=None, headers=None, **kwargs): predicate=ResumableUploadSession()._get_retry_predicate(), initial=0.001, ) - config = ResumableUploadConfig(chunk_size=4, retry=retry_cfg) + config = ResumableUploadConfig(chunk_size=4) session = ResumableUploadSession( upload_url="https://api.example.com/init", config=config, ) - session.upload(stream=b"012345", transport=session_transport) + session.upload(stream=b"012345", transport=session_transport, retry=retry_cfg) # Verify no data loss occurred: server must receive all 6 bytes (b"012345"), not truncated b"0123" assert bytes(server_received_bytes) == b"012345" @@ -1705,17 +1678,78 @@ def test_sync_streaming_retry_and_recovery_final(): predicate=ResumableUploadSession()._get_retry_predicate(), initial=0.001, ) - config = ResumableUploadConfig( - response_type=DummyResponse, - retry=streaming_retry, - ) session = ResumableUploadSession( upload_url="https://api.example.com/init", - config=config, + response_type=DummyResponse, + ) + assert ( + session._get_streaming_retry(retry_override=streaming_retry) is streaming_retry ) - assert session._get_streaming_retry() is streaming_retry - result = session.upload(stream=b"data", transport=session_transport) + result = session.upload( + stream=b"data", transport=session_transport, retry=streaming_retry + ) assert isinstance(result, DummyResponse) assert result.name == "already_done.txt" assert session.finished is True + + +def test_sync_method_override_arguments() -> None: + """Verifies content_type and on_progress overrides on initiate, upload, and resume.""" + # 1. initiate with content_type override + start_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-URL": "https://upload.example.com/123", + }, + content=b"", + ) + transport1 = mock.Mock(spec=requests.Session) + transport1.request.return_value = start_resp + session1 = ResumableUploadSession(upload_url="https://api.example.com/start") + session1.initiate(transport=transport1, content_type="text/plain") + assert session1._content_type == "text/plain" + + # 2. upload with content_type and on_progress overrides + progress_events: List[UploadProgress] = [] + chunk_resp = mock.Mock( + ok=True, + status_code=200, + headers={"X-Goog-Upload-Status": "final"}, + content=b"{}", + ) + transport2 = mock.Mock(spec=requests.Session) + transport2.request.side_effect = [start_resp, chunk_resp] + session2 = ResumableUploadSession(upload_url="https://api.example.com/start") + session2.upload( + stream=b"data", + transport=transport2, + content_type="text/csv", + on_progress=lambda p: progress_events.append(p), + ) + assert session2._content_type == "text/csv" + assert len(progress_events) == 2 + + # 3. resume with on_progress override + resume_events: List[UploadProgress] = [] + query_resp = mock.Mock( + ok=True, + status_code=200, + headers={ + "X-Goog-Upload-Status": "active", + "X-Goog-Upload-Size-Received": "0", + }, + content=b"", + ) + transport3 = mock.Mock(spec=requests.Session) + transport3.request.side_effect = [query_resp, chunk_resp] + session3 = ResumableUploadSession() + session3.resume( + upload_url="https://upload.example.com/123", + stream=b"data", + transport=transport3, + on_progress=lambda p: resume_events.append(p), + ) + assert len(resume_events) == 2 From fc3797e786f721ea55e43640ec6e0cd7bac8e546 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 17:15:33 +0000 Subject: [PATCH 34/43] add entry for google-cloud-spanner-dbapi-driver to satisfy regeneration check --- librarian.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/librarian.yaml b/librarian.yaml index e33e3e95193c..bfa9f8eb6c8a 100644 --- a/librarian.yaml +++ b/librarian.yaml @@ -2000,6 +2000,10 @@ libraries: - python-gapic-name=spanner metadata_name_override: spanner default_version: v1 + - name: google-cloud-spanner-dbapi-driver + version: 0.0.0 + python: + library_type: INTEGRATION - name: google-cloud-speech version: 2.40.0 apis: From cc14bbea48dfc89442e0a1449e6fcd6bac1b1105 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 17:54:39 +0000 Subject: [PATCH 35/43] update docstring --- .../google/api_core/resumable_transfer/common.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/common.py b/packages/google-api-core/google/api_core/resumable_transfer/common.py index 9335f45cc911..c56d96b70b32 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/common.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/common.py @@ -103,7 +103,12 @@ class ResumableUploadConfig: stall_minimum_rate: Minimum transfer rate in bytes per second. Defaults to 64 KiB/s. stall_timeout: Stall duration threshold in seconds. Defaults to 120s. headers: Additional HTTP headers dispatched exclusively with start request. - deadline: Overall global deadline for the upload process. + deadline: Optional overall wall-clock deadline for the entire upload process. + When set, each HTTP request timeout is trimmed to the remaining time before + the deadline, and DeadlineExceeded is raised immediately when the deadline + elapses (even on healthy streams). When None (default), transfer duration is + governed by stall control (stall_minimum_rate and stall_timeout), allowing + healthy streams transferring above the minimum rate to continue indefinitely. """ chunk_size: int = DEFAULT_CHUNK_SIZE From 9007660becfd9796bf9955864e9e5acd62357a3e Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 17:55:46 +0000 Subject: [PATCH 36/43] remove diff --- .../.repo-metadata.json | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/google-cloud-spanner-dbapi-driver/.repo-metadata.json b/packages/google-cloud-spanner-dbapi-driver/.repo-metadata.json index 6008b776651b..293f7836f3a9 100644 --- a/packages/google-cloud-spanner-dbapi-driver/.repo-metadata.json +++ b/packages/google-cloud-spanner-dbapi-driver/.repo-metadata.json @@ -1,16 +1,9 @@ { - "api_description": "Google Cloud Spanner DBAPI 2.0 Driver", - "api_id": "spanner.googleapis.com", - "api_shortname": "spanner", - "client_documentation": "https://cloud.google.com/python/docs/reference/spanner-dbapi-driver/latest", - "default_version": "v1", - "distribution_name": "google-cloud-spanner-dbapi-driver", - "issue_tracker": "https://github.com/googleapis/google-cloud-python/issues", - "language": "python", - "library_type": "MANUAL", - "name": "spanner-dbapi-driver", - "name_pretty": "Spanner DBAPI Driver", - "product_documentation": "https://cloud.google.com/spanner/", - "release_level": "preview", - "repo": "googleapis/google-cloud-python" -} + "client_documentation": "https://cloud.google.com/python/docs/reference/google-cloud-spanner-dbapi-driver/latest", + "distribution_name": "google-cloud-spanner-dbapi-driver", + "language": "python", + "library_type": "INTEGRATION", + "name": "google-cloud-spanner-dbapi-driver", + "release_level": "preview", + "repo": "googleapis/google-cloud-python" +} \ No newline at end of file From e797b15db5aa1db16259aad76199f16d7a514c0b Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 18:42:28 +0000 Subject: [PATCH 37/43] fix build --- packages/google-cloud-spanner-dbapi-driver/CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 packages/google-cloud-spanner-dbapi-driver/CHANGELOG.md diff --git a/packages/google-cloud-spanner-dbapi-driver/CHANGELOG.md b/packages/google-cloud-spanner-dbapi-driver/CHANGELOG.md new file mode 100644 index 000000000000..eeaeefeafa75 --- /dev/null +++ b/packages/google-cloud-spanner-dbapi-driver/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +[PyPI History][1] + +[1]: https://pypi.org/project/google-cloud-spanner-dbapi-driver/#history From a72798de17b8af9d32586321184c6d841814bdda Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 18:57:34 +0000 Subject: [PATCH 38/43] fix build --- packages/google-cloud-spanner-dbapi-driver/docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) create mode 120000 packages/google-cloud-spanner-dbapi-driver/docs/CHANGELOG.md diff --git a/packages/google-cloud-spanner-dbapi-driver/docs/CHANGELOG.md b/packages/google-cloud-spanner-dbapi-driver/docs/CHANGELOG.md new file mode 120000 index 000000000000..04c99a55caae --- /dev/null +++ b/packages/google-cloud-spanner-dbapi-driver/docs/CHANGELOG.md @@ -0,0 +1 @@ +../CHANGELOG.md \ No newline at end of file From 1e23266974d1a0db08901f819274299be70adf35 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 16 Sep 2026 19:04:09 +0000 Subject: [PATCH 39/43] fix build --- packages/google-cloud-spanner-dbapi-driver/docs/index.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/google-cloud-spanner-dbapi-driver/docs/index.rst b/packages/google-cloud-spanner-dbapi-driver/docs/index.rst index 98ebacf8044d..c150aae41bfc 100644 --- a/packages/google-cloud-spanner-dbapi-driver/docs/index.rst +++ b/packages/google-cloud-spanner-dbapi-driver/docs/index.rst @@ -13,7 +13,6 @@ Changelog For a list of all ``google-cloud-spanner-dbapi`` releases: .. toctree:: - :maxdepth: 2 + :maxdepth: 2 -.. toctree:: - :hidden: + CHANGELOG From 0d51195af329fb6d51e4a91bb56b407e85b3ed5a Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 17 Sep 2026 15:00:50 +0000 Subject: [PATCH 40/43] address review feedback --- .../api_core/resumable_transfer/common.py | 6 +- .../api_core/resumable_transfer/upload.py | 113 ++++++++++++------ .../resumable_transfer/upload_async.py | 97 +++++++++------ .../asyncio/test_resumable_transfer_async.py | 109 +++++++++++++++-- .../tests/unit/test_resumable_transfer.py | 98 +++++++++++++-- 5 files changed, 328 insertions(+), 95 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/common.py b/packages/google-api-core/google/api_core/resumable_transfer/common.py index c56d96b70b32..d1ac035476b6 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/common.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/common.py @@ -87,11 +87,11 @@ class UploadProgress: state: ProgressState -# HTTP status codes indicating transient retryable errors +# HTTP status codes indicating transient retryable errors (Category 1) RETRYABLE_STATUS_CODES = (408, 429, 500, 502, 503, 504) -# HTTP status codes indicating state consistency errors requiring recovery -RECOVERABLE_STATUS_CODES = (400, 409, 412, 416) +# HTTP status codes indicating state consistency errors requiring recovery (Category 2) +RECOVERABLE_STATUS_CODES = (400, 412, 416) @dataclasses.dataclass diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index 29cfe482b954..b30dda07de24 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -115,7 +115,12 @@ def __init__( transport: Optional requests session. content_type: Optional MIME type of the stream payload. response_type: Optional message class, callable deserializer, or None. - start_retry: Optional unary Retry policy for the start request. + start_retry: Optional retry configuration (``google.api_core.retry.Retry``) + for the initial session creation request. Use this to customize + exponential backoff timing (such as ``Retry(initial=1.0, maximum=60.0)``) + or to supply a custom ``predicate`` function for API-specific transient + errors. Terminal errors (such as ``DeadlineExceeded`` or + ``TransferStalledError``) are never retried. start_timeout: Optional timeout in seconds for the start request. """ self._config = config or ResumableUploadConfig() @@ -259,24 +264,28 @@ def _get_start_timeout(self, timeout_override: Optional[float] = None) -> float: return min(timeout, remaining) return timeout - def _get_retry_predicate(self, is_start: bool = False) -> Callable[[Any], bool]: + def _get_retry_predicate( + self, + is_start: bool = False, + custom_predicate: Optional[Callable[[Exception], bool]] = None, + ) -> Callable[[Exception], bool]: """Returns a predicate function for determining if an exception is retryable. Args: is_start: If True, only transient status codes (RETRYABLE_STATUS_CODES) - are retried. If False (chunk transfer phase), state consistency - status codes (RECOVERABLE_STATUS_CODES) are also retried via recovery. + are retried. If False (transmitting and finalizing states), state + consistency errors (RECOVERABLE_STATUS_CODES and + MissingStatusHeaderError) are also retried via recovery. + custom_predicate: Optional callable taking an exception and returning True + if the error should be retried (from a user-supplied Retry instance). + When provided, this function is evaluated for non-terminal errors + while automatically preserving resumable upload state recovery. Returns: A callable accepting an exception and returning a boolean. """ - allowed_codes = ( - common.RETRYABLE_STATUS_CODES - if is_start - else (common.RECOVERABLE_STATUS_CODES + common.RETRYABLE_STATUS_CODES) - ) - def should_retry(exc: Any) -> bool: + def should_retry(exc: Exception) -> bool: if isinstance( exc, ( @@ -287,8 +296,19 @@ def should_retry(exc: Any) -> bool: ), ): return False - if isinstance(exc, exceptions.MissingStatusHeaderError): + if not is_start and ( + isinstance(exc, exceptions.MissingStatusHeaderError) + or ( + isinstance(exc, exceptions.GoogleAPICallError) + and exc.code in common.RECOVERABLE_STATUS_CODES + ) + ): return True + if ( + custom_predicate is not None + and custom_predicate is not google.api_core.retry.if_transient_error + ): + return bool(custom_predicate(exc)) if isinstance(exc, requests.exceptions.RequestException): if isinstance( exc, @@ -300,7 +320,7 @@ def should_retry(exc: Any) -> bool: ): return True if isinstance(exc, exceptions.GoogleAPICallError): - return exc.code in allowed_codes + return exc.code in common.RETRYABLE_STATUS_CODES return False return should_retry @@ -318,7 +338,11 @@ def _get_retry( """ candidate = retry_override or self._start_retry if candidate is not None: - return candidate + return candidate.with_predicate( + self._get_retry_predicate( + is_start=True, custom_predicate=candidate._predicate + ) + ) return google.api_core.retry.Retry( predicate=self._get_retry_predicate(is_start=True) ) @@ -338,10 +362,13 @@ def _get_streaming_retry( Configured or default StreamingRetry instance. """ if retry_override is not None: + wrapped_pred = self._get_retry_predicate( + is_start=False, custom_predicate=retry_override._predicate + ) if isinstance(retry_override, google.api_core.retry.StreamingRetry): - return retry_override + return retry_override.with_predicate(wrapped_pred) return google.api_core.retry.StreamingRetry( - predicate=retry_override._predicate, + predicate=wrapped_pred, initial=retry_override._initial, maximum=retry_override._maximum, multiplier=retry_override._multiplier, @@ -482,7 +509,9 @@ def initiate( size: Total size of payload in bytes, if known. progress_queue: Optional list buffering UploadProgress snapshots. content_type: Optional MIME type override of the payload. - retry: Optional unary Retry policy override for the start request. + retry: Optional retry configuration (``Retry``) for this session initiation + call. Overrides ``start_retry`` if provided. Use this to customize + backoff timing or add custom retryable exceptions. timeout: Optional per-request timeout override in seconds. Returns: @@ -695,6 +724,7 @@ def _transmit_all_chunks( yield progress_queue.pop(0) final_resp: Optional[requests.Response] = None + retry_policy = self._get_streaming_retry(retry_override=retry) def attempt_stream() -> Generator[common.UploadProgress, None, None]: nonlocal final_resp @@ -725,30 +755,13 @@ def attempt_stream() -> Generator[common.UploadProgress, None, None]: timeout=timeout, ) except Exception as exc: - is_recoverable = ( - isinstance(exc, exceptions.GoogleAPICallError) - and exc.code - in ( - common.RECOVERABLE_STATUS_CODES - + common.RETRYABLE_STATUS_CODES - ) - ) or isinstance( - exc, - ( - exceptions.MissingStatusHeaderError, - requests.exceptions.ConnectionError, - requests.exceptions.ChunkedEncodingError, - requests.exceptions.Timeout, - ), - ) - if is_recoverable: + if retry_policy._predicate(exc): self._needs_recovery = True raise while progress_queue: yield progress_queue.pop(0) try: - retry_policy = self._get_streaming_retry(retry_override=retry) retryable_stream = retry_policy(attempt_stream) yield from retryable_stream() except requests.exceptions.Timeout as exc: @@ -786,7 +799,13 @@ def upload( size: Total stream size in bytes, if known. transport: Optional requests session. content_type: Optional MIME type of the stream payload. - retry: Optional retry policy override for chunk transmission. + retry: Optional retry configuration (``Retry`` or ``StreamingRetry``) for + chunk upload requests. Use this to customize exponential backoff + timing between chunk retries or to supply a custom ``predicate`` for + API-specific transient errors. Protocol recovery (such as server offset + synchronization on missing status headers) is preserved automatically, + and terminal errors (such as ``DeadlineExceeded`` or + ``TransferStalledError``) are never retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. @@ -831,7 +850,13 @@ def iter_upload( size: Total stream size in bytes, if known. transport: Optional requests session. content_type: Optional MIME type of the stream payload. - retry: Optional retry policy override for chunk transmission. + retry: Optional retry configuration (``Retry`` or ``StreamingRetry``) for + chunk upload requests. Use this to customize exponential backoff + timing between chunk retries or to supply a custom ``predicate`` for + API-specific transient errors. Protocol recovery (such as server offset + synchronization on missing status headers) is preserved automatically, + and terminal errors (such as ``DeadlineExceeded`` or + ``TransferStalledError``) are never retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. @@ -889,7 +914,13 @@ def resume( size: Total size of the payload in bytes, if known. chunk_size: Optional chunk size override in bytes. transport: Optional requests session. - retry: Optional retry policy override for chunk transmission. + retry: Optional retry configuration (``Retry`` or ``StreamingRetry``) for + chunk upload requests. Use this to customize exponential backoff + timing between chunk retries or to supply a custom ``predicate`` for + API-specific transient errors. Protocol recovery (such as server offset + synchronization on missing status headers) is preserved automatically, + and terminal errors (such as ``DeadlineExceeded`` or + ``TransferStalledError``) are never retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. @@ -934,7 +965,13 @@ def iter_resume( size: Total size of the payload in bytes, if known. chunk_size: Optional chunk size override in bytes. transport: Optional requests session. - retry: Optional retry policy override for chunk transmission. + retry: Optional retry configuration (``Retry`` or ``StreamingRetry``) for + chunk upload requests. Use this to customize exponential backoff + timing between chunk retries or to supply a custom ``predicate`` for + API-specific transient errors. Protocol recovery (such as server offset + synchronization on missing status headers) is preserved automatically, + and terminal errors (such as ``DeadlineExceeded`` or + ``TransferStalledError``) are never retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index 06a270429646..cb046fccbfd1 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -171,7 +171,12 @@ def __init__( transport: Optional aiohttp.ClientSession. content_type: Optional MIME type of the stream payload. response_type: Optional message class, callable deserializer, or None. - start_retry: Optional unary AsyncRetry policy for the start request. + start_retry: Optional retry configuration (``google.api_core.retry.AsyncRetry``) + for the initial session creation request. Use this to customize + exponential backoff timing (such as ``AsyncRetry(initial=1.0, maximum=60.0)``) + or to supply a custom ``predicate`` function for API-specific transient + errors. Terminal errors (such as ``DeadlineExceeded`` or + ``TransferStalledError``) are never retried. start_timeout: Optional timeout in seconds for the start request. """ self._config = config or ResumableUploadConfig() @@ -311,24 +316,28 @@ def _get_start_timeout(self, timeout_override: Optional[float] = None) -> float: return min(timeout, remaining) return timeout - def _get_retry_predicate(self, is_start: bool = False) -> Callable[[Any], bool]: + def _get_retry_predicate( + self, + is_start: bool = False, + custom_predicate: Optional[Callable[[Exception], bool]] = None, + ) -> Callable[[Exception], bool]: """Returns a predicate function for determining if an exception is retryable. Args: is_start: If True, only transient status codes (RETRYABLE_STATUS_CODES) - are retried. If False (chunk transfer phase), state consistency - status codes (RECOVERABLE_STATUS_CODES) are also retried via recovery. + are retried. If False (transmitting and finalizing states), state + consistency errors (RECOVERABLE_STATUS_CODES and + MissingStatusHeaderError) are also retried via recovery. + custom_predicate: Optional callable taking an exception and returning True + if the error should be retried (from a user-supplied AsyncRetry instance). + When provided, this function is evaluated for non-terminal errors + while automatically preserving resumable upload state recovery. Returns: A callable accepting an exception and returning a boolean. """ - allowed_codes = ( - common.RETRYABLE_STATUS_CODES - if is_start - else (common.RECOVERABLE_STATUS_CODES + common.RETRYABLE_STATUS_CODES) - ) - def should_retry(exc: Any) -> bool: + def should_retry(exc: Exception) -> bool: if isinstance( exc, ( @@ -339,12 +348,23 @@ def should_retry(exc: Any) -> bool: ), ): return False - if isinstance(exc, exceptions.MissingStatusHeaderError): + if not is_start and ( + isinstance(exc, exceptions.MissingStatusHeaderError) + or ( + isinstance(exc, exceptions.GoogleAPICallError) + and exc.code in common.RECOVERABLE_STATUS_CODES + ) + ): return True + if ( + custom_predicate is not None + and custom_predicate is not google.api_core.retry.if_transient_error + ): + return bool(custom_predicate(exc)) if _HAS_AIOHTTP and isinstance(exc, aiohttp.ClientError): return True if isinstance(exc, exceptions.GoogleAPICallError): - return exc.code in allowed_codes + return exc.code in common.RETRYABLE_STATUS_CODES return False return should_retry @@ -362,7 +382,11 @@ def _get_async_retry( """ candidate = retry_override or self._start_retry if candidate is not None: - return candidate + return candidate.with_predicate( + self._get_retry_predicate( + is_start=True, custom_predicate=candidate._predicate + ) + ) return google.api_core.retry.AsyncRetry( predicate=self._get_retry_predicate(is_start=True) ) @@ -385,10 +409,13 @@ def _get_async_streaming_retry( Configured or default AsyncStreamingRetry instance. """ if retry_override is not None: + wrapped_pred = self._get_retry_predicate( + is_start=False, custom_predicate=retry_override._predicate + ) if isinstance(retry_override, google.api_core.retry.AsyncStreamingRetry): - return retry_override + return retry_override.with_predicate(wrapped_pred) return google.api_core.retry.AsyncStreamingRetry( - predicate=retry_override._predicate, + predicate=wrapped_pred, initial=retry_override._initial, maximum=retry_override._maximum, multiplier=retry_override._multiplier, @@ -417,7 +444,9 @@ async def initiate( size: Total stream size in bytes, if known. progress_queue: Optional queue to receive progress event. content_type: Optional MIME type override of the payload. - retry: Optional unary AsyncRetry policy override for the start request. + retry: Optional retry configuration (``AsyncRetry``) for this session + initiation call. Overrides ``start_retry`` if provided. Use this to + customize backoff timing or add custom retryable exceptions. timeout: Optional per-request timeout override in seconds. Returns: @@ -733,6 +762,7 @@ async def _transmit_all_chunks( Tuple of (status code, headers, body bytes) of the final server response, or None. """ final_resp_tuple: Optional[Tuple[int, Mapping[str, str], bytes]] = None + retry_policy = self._get_async_streaming_retry(retry_override=retry) async def attempt_stream() -> AsyncGenerator[None, None]: nonlocal final_resp_tuple @@ -759,27 +789,12 @@ async def attempt_stream() -> AsyncGenerator[None, None]: timeout=timeout, ) except Exception as exc: - is_recoverable = ( - isinstance(exc, exceptions.GoogleAPICallError) - and exc.code - in ( - common.RECOVERABLE_STATUS_CODES - + common.RETRYABLE_STATUS_CODES - ) - ) or isinstance( - exc, - ( - exceptions.MissingStatusHeaderError, - aiohttp.ClientError, - ), - ) - if is_recoverable: + if retry_policy._predicate(exc): self._needs_recovery = True raise yield try: - retry_policy = self._get_async_streaming_retry(retry_override=retry) retryable_stream = retry_policy(attempt_stream) stream_gen = await retryable_stream() async for _ in stream_gen: @@ -822,7 +837,14 @@ def upload( size: Total stream size in bytes, if known. transport: Optional aiohttp client session. content_type: Optional MIME type of the stream payload. - retry: Optional retry policy override for chunk transmission. + retry: Optional retry configuration (``AsyncRetry`` or + ``AsyncStreamingRetry``) for chunk upload requests. Use this to + customize exponential backoff timing between chunk retries or to + supply a custom ``predicate`` for API-specific transient errors. + Protocol recovery (such as server offset synchronization on + missing status headers) is preserved automatically, and terminal + errors (such as ``DeadlineExceeded`` or ``TransferStalledError``) + are never retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. @@ -905,7 +927,14 @@ def resume( size: Total stream size in bytes, if known. chunk_size: Optional chunk size override in bytes. transport: Optional aiohttp client session. - retry: Optional retry policy override for chunk transmission. + retry: Optional retry configuration (``AsyncRetry`` or + ``AsyncStreamingRetry``) for chunk upload requests. Use this to + customize exponential backoff timing between chunk retries or to + supply a custom ``predicate`` for API-specific transient errors. + Protocol recovery (such as server offset synchronization on + missing status headers) is preserved automatically, and terminal + errors (such as ``DeadlineExceeded`` or ``TransferStalledError``) + are never retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. diff --git a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py index 0f2a6043c844..36194c911d6c 100644 --- a/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py +++ b/packages/google-api-core/tests/asyncio/test_resumable_transfer_async.py @@ -699,8 +699,10 @@ async def test_async_recoverable_status_code_triggers_recovery() -> None: }, body=b"", ) - # Chunk 1 returns 409 Conflict - chunk_conflict = DummyAsyncResponse(status=409, headers={}, body=b"Conflict") + # Chunk 1 returns Category 2 recoverable error (412 Precondition Failed) + chunk_precondition_failed = DummyAsyncResponse( + status=412, headers={}, body=b"Precondition Failed" + ) # Recovery query returns confirmed committed offset 0 query_resp = DummyAsyncResponse( status=200, @@ -718,7 +720,7 @@ async def test_async_recoverable_status_code_triggers_recovery() -> None: ) async_transport = DummyAsyncSession( - [start_resp, chunk_conflict, query_resp, chunk_success] + [start_resp, chunk_precondition_failed, query_resp, chunk_success] ) session = AsyncResumableUploadSession( upload_url="https://api.example.com/start", @@ -1110,14 +1112,28 @@ async def test_async_retry_branches() -> None: assert pred_start(exceptions.TransferStalledError("stalled")) is False assert pred_start(exceptions.UploadCancelledError("cancelled")) is False assert pred_start(exceptions.UnseekableStreamError("unseekable")) is False - assert pred_start(exceptions.MissingStatusHeaderError("missing")) is True + assert pred_start(exceptions.MissingStatusHeaderError("missing")) is False + assert pred_transfer(exceptions.MissingStatusHeaderError("missing")) is True assert pred_start(aiohttp.ClientError("network error")) is True assert pred_start(exceptions.from_http_status(503, "Service Unavailable")) is True assert pred_start(exceptions.from_http_status(400, "Bad Request")) is False + assert pred_start(exceptions.from_http_status(412, "Precondition Failed")) is False + assert ( + pred_start(exceptions.from_http_status(416, "Range Not Satisfiable")) is False + ) + assert pred_start(exceptions.from_http_status(409, "Conflict")) is False assert pred_start(RuntimeError("runtime")) is False - # Recoverable status code 500 is retryable during transfer + # Category 1 and Category 2 status codes are retryable during transfer assert pred_transfer(exceptions.from_http_status(500, "Internal Error")) is True + assert pred_transfer(exceptions.from_http_status(400, "Bad Request")) is True + assert ( + pred_transfer(exceptions.from_http_status(412, "Precondition Failed")) is True + ) + assert ( + pred_transfer(exceptions.from_http_status(416, "Range Not Satisfiable")) is True + ) + assert pred_transfer(exceptions.from_http_status(409, "Conflict")) is False # Default retry resolution default_unary = session._get_async_retry() @@ -1126,25 +1142,94 @@ async def test_async_retry_branches() -> None: default_stream = session._get_async_streaming_retry() assert isinstance(default_stream, google.api_core.retry.AsyncStreamingRetry) - # Custom AsyncRetry resolution via start_retry and retry_override - custom_unary = google.api_core.retry.AsyncRetry(initial=0.5) + class CustomApiError(Exception): + """Example API-specific transient exception provided by a caller.""" + + # ------------------------------------------------------------------------- + # Scenario 1: User provides a custom predicate to retry an API-specific error + # ------------------------------------------------------------------------- + custom_unary = google.api_core.retry.AsyncRetry( + initial=0.5, + predicate=lambda exc: isinstance(exc, CustomApiError), + ) session_unary = AsyncResumableUploadSession( upload_url="https://api.example.com/start", start_retry=custom_unary, ) - assert session_unary._get_async_retry() is custom_unary + resolved_unary = session_unary._get_async_retry() + + assert isinstance(resolved_unary, google.api_core.retry.AsyncRetry) + assert resolved_unary._initial == 0.5 + # User's custom exception is retried, while unrelated errors are not + assert resolved_unary._predicate(CustomApiError("rate limit")) is True + assert resolved_unary._predicate(ValueError("invalid input")) is False + # Terminal errors must always return False regardless of custom predicate + assert ( + resolved_unary._predicate(exceptions.DeadlineExceeded("deadline expired")) + is False + ) + + # ------------------------------------------------------------------------- + # Scenario 2: Unary AsyncRetry passed to chunk transfer converts to AsyncStreamingRetry + # ------------------------------------------------------------------------- converted_stream = session_unary._get_async_streaming_retry( retry_override=custom_unary ) assert isinstance(converted_stream, google.api_core.retry.AsyncStreamingRetry) assert converted_stream._initial == 0.5 + assert converted_stream._predicate(CustomApiError("rate limit")) is True + # Category 2 recovery (400, 412, 416, MissingStatusHeaderError) is preserved during chunk transfer + missing_header_error = exceptions.MissingStatusHeaderError( + "Missing X-Goog-Upload-Status" + ) + assert converted_stream._predicate(missing_header_error) is True + assert ( + converted_stream._predicate( + exceptions.from_http_status(412, "Precondition Failed") + ) + is True + ) - # Custom AsyncStreamingRetry resolution via retry_override - custom_stream = google.api_core.retry.AsyncStreamingRetry(initial=0.25) + # ------------------------------------------------------------------------- + # Scenario 3: Restrictive custom predicate still preserves Category 2 recovery + # ------------------------------------------------------------------------- + restrictive_stream = google.api_core.retry.AsyncStreamingRetry( + initial=0.25, + predicate=lambda exc: False, + ) + resolved_stream = session._get_async_streaming_retry( + retry_override=restrictive_stream + ) + assert isinstance(resolved_stream, google.api_core.retry.AsyncStreamingRetry) + assert resolved_stream._initial == 0.25 + # Category 2 errors (400, 412, 416, MissingStatusHeaderError) return True so + # server offset synchronization is preserved + assert ( + resolved_stream._predicate(exceptions.from_http_status(400, "Bad Request")) + is True + ) + assert ( + resolved_stream._predicate( + exceptions.from_http_status(412, "Precondition Failed") + ) + is True + ) + assert ( + resolved_stream._predicate( + exceptions.from_http_status(416, "Range Not Satisfiable") + ) + is True + ) + assert resolved_stream._predicate(missing_header_error) is True + # Unretriable HTTP status codes (such as 409 Conflict) and non-protocol errors + # follow the predicate and return False assert ( - session._get_async_streaming_retry(retry_override=custom_stream) - is custom_stream + resolved_stream._predicate(exceptions.from_http_status(409, "Conflict")) + is False ) + bad_gateway_error = exceptions.from_http_status(502, "Bad Gateway") + assert resolved_stream._predicate(bad_gateway_error) is False + assert resolved_stream._predicate(RuntimeError("unexpected crash")) is False def test_async_transport_missing_errors() -> None: diff --git a/packages/google-api-core/tests/unit/test_resumable_transfer.py b/packages/google-api-core/tests/unit/test_resumable_transfer.py index 019b86620dd5..9154b9ea6820 100644 --- a/packages/google-api-core/tests/unit/test_resumable_transfer.py +++ b/packages/google-api-core/tests/unit/test_resumable_transfer.py @@ -869,11 +869,18 @@ def test_sync_retry_predicate_branches(): assert pred_transfer(TransferStalledError("stalled")) is False assert pred_transfer(UploadCancelledError("cancelled")) is False assert pred_transfer(MissingStatusHeaderError("missing")) is True + assert pred_start(MissingStatusHeaderError("missing")) is False assert pred_transfer(requests.exceptions.ConnectionError("conn")) is True assert pred_transfer(requests.exceptions.ChunkedEncodingError("chunked")) is True assert pred_transfer(exceptions.from_http_status(503, "503")) is True assert pred_transfer(exceptions.from_http_status(400, "400")) is True + assert pred_transfer(exceptions.from_http_status(412, "412")) is True + assert pred_transfer(exceptions.from_http_status(416, "416")) is True assert pred_start(exceptions.from_http_status(400, "400")) is False + assert pred_start(exceptions.from_http_status(412, "412")) is False + assert pred_start(exceptions.from_http_status(416, "416")) is False + assert pred_transfer(exceptions.from_http_status(409, "409")) is False + assert pred_start(exceptions.from_http_status(409, "409")) is False assert pred_transfer(exceptions.from_http_status(403, "403")) is False assert pred_transfer(TypeError("other")) is False @@ -1027,14 +1034,90 @@ def test_sync_naive_deadline_tz(): def test_sync_get_retry_start_and_override(): - ret_start = mock.Mock(spec=google.api_core.retry.Retry) - ret_override = mock.Mock(spec=google.api_core.retry.Retry) + """Verifies how user-supplied Retry objects are resolved and wrapped.""" + + class CustomApiError(Exception): + """Example API-specific transient exception provided by a caller.""" + + # ------------------------------------------------------------------------- + # Scenario 1: User provides a custom predicate to retry an API-specific error + # ------------------------------------------------------------------------- + custom_start_retry = google.api_core.retry.Retry( + initial=0.5, + predicate=lambda exc: isinstance(exc, CustomApiError), + ) session = ResumableUploadSession( upload_url="https://api.example.com/init", - start_retry=ret_start, + start_retry=custom_start_retry, + ) + resolved_start = session._get_retry() + + # Backoff settings (0.5s initial delay) are preserved + assert resolved_start._initial == 0.5 + # The user's custom exception is retried, while unrelated errors are not + assert resolved_start._predicate(CustomApiError("rate limit")) is True + assert resolved_start._predicate(ValueError("invalid input")) is False + + # ------------------------------------------------------------------------- + # Scenario 2: Terminal errors are blocked even if custom predicate returns True + # ------------------------------------------------------------------------- + overly_broad_retry = google.api_core.retry.Retry( + initial=0.25, + predicate=lambda exc: True, + ) + resolved_override = session._get_retry(retry_override=overly_broad_retry) + + assert resolved_override._initial == 0.25 + # Terminal transfer errors must never be retried, preventing infinite loops + assert ( + resolved_override._predicate(exceptions.DeadlineExceeded("deadline expired")) + is False + ) + assert ( + resolved_override._predicate(exceptions.TransferStalledError("upload stalled")) + is False ) - assert session._get_retry() is ret_start - assert session._get_retry(retry_override=ret_override) is ret_override + + # ------------------------------------------------------------------------- + # Scenario 3: Category 2 recovery (400, 412, 416, MissingStatusHeaderError) + # is preserved during chunk transfer + # ------------------------------------------------------------------------- + restrictive_chunk_retry = google.api_core.retry.StreamingRetry( + initial=0.1, + predicate=lambda exc: False, + ) + resolved_chunk = session._get_streaming_retry( + retry_override=restrictive_chunk_retry + ) + + assert resolved_chunk._initial == 0.1 + # Category 2 recovery triggers (400, 412, 416, and missing status header) + # must still return True during chunk transfer so the client can query + # server offset and synchronize state + assert ( + resolved_chunk._predicate(exceptions.from_http_status(400, "Bad Request")) + is True + ) + assert ( + resolved_chunk._predicate( + exceptions.from_http_status(412, "Precondition Failed") + ) + is True + ) + assert ( + resolved_chunk._predicate( + exceptions.from_http_status(416, "Range Not Satisfiable") + ) + is True + ) + missing_header_error = MissingStatusHeaderError("Missing X-Goog-Upload-Status") + assert resolved_chunk._predicate(missing_header_error) is True + # Unretriable HTTP status codes (such as 409 Conflict) and non-protocol errors + # follow the predicate and return False + assert ( + resolved_chunk._predicate(exceptions.from_http_status(409, "Conflict")) is False + ) + assert resolved_chunk._predicate(RuntimeError("unexpected crash")) is False def test_sync_stall_control_with_deadline(): @@ -1682,9 +1765,8 @@ def test_sync_streaming_retry_and_recovery_final(): upload_url="https://api.example.com/init", response_type=DummyResponse, ) - assert ( - session._get_streaming_retry(retry_override=streaming_retry) is streaming_retry - ) + resolved_streaming = session._get_streaming_retry(retry_override=streaming_retry) + assert resolved_streaming._initial == 0.001 result = session.upload( stream=b"data", transport=session_transport, retry=streaming_retry From 29cdf19f730883fb00897caf43b1676949517a00 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 17 Sep 2026 15:12:58 +0000 Subject: [PATCH 41/43] update docstrings --- .../api_core/resumable_transfer/upload.py | 48 ++++++++++++++----- .../resumable_transfer/upload_async.py | 38 +++++++++++---- 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload.py b/packages/google-api-core/google/api_core/resumable_transfer/upload.py index b30dda07de24..a05edbfd7c54 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload.py @@ -119,8 +119,9 @@ def __init__( for the initial session creation request. Use this to customize exponential backoff timing (such as ``Retry(initial=1.0, maximum=60.0)``) or to supply a custom ``predicate`` function for API-specific transient - errors. Terminal errors (such as ``DeadlineExceeded`` or - ``TransferStalledError``) are never retried. + errors. Terminal errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. start_timeout: Optional timeout in seconds for the start request. """ self._config = config or ResumableUploadConfig() @@ -280,6 +281,9 @@ def _get_retry_predicate( if the error should be retried (from a user-supplied Retry instance). When provided, this function is evaluated for non-terminal errors while automatically preserving resumable upload state recovery. + Terminal errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. Returns: A callable accepting an exception and returning a boolean. @@ -331,7 +335,10 @@ def _get_retry( """Resolves unary Retry policy for start requests. Args: - retry_override: Optional unary Retry policy override for the start request. + retry_override: Optional unary Retry policy override for the start + request. Terminal errors (``DeadlineExceeded``, + ``TransferStalledError``, ``UploadCancelledError``, and + ``UnseekableStreamError``) are never retried. Returns: Configured or default unary Retry instance. @@ -357,6 +364,10 @@ def _get_streaming_retry( Args: retry_override: Optional retry policy override for chunk transmission. + Protocol recovery is preserved automatically, and terminal errors + (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. Returns: Configured or default StreamingRetry instance. @@ -511,7 +522,10 @@ def initiate( content_type: Optional MIME type override of the payload. retry: Optional retry configuration (``Retry``) for this session initiation call. Overrides ``start_retry`` if provided. Use this to customize - backoff timing or add custom retryable exceptions. + backoff timing or add custom retryable exceptions. Terminal errors + (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-request timeout override in seconds. Returns: @@ -708,7 +722,11 @@ def _transmit_all_chunks( stream_obj: Binary stream yielding upload chunks. computed_size: Total payload size in bytes if known. progress_queue: Optional list buffering UploadProgress snapshots. - retry: Optional retry policy override for chunk transmission. + retry: Optional retry policy override for chunk transmission. Protocol + recovery is preserved automatically, and terminal errors + (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-attempt timeout ceiling in seconds. Yields: @@ -804,8 +822,9 @@ def upload( timing between chunk retries or to supply a custom ``predicate`` for API-specific transient errors. Protocol recovery (such as server offset synchronization on missing status headers) is preserved automatically, - and terminal errors (such as ``DeadlineExceeded`` or - ``TransferStalledError``) are never retried. + and terminal errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. @@ -855,8 +874,9 @@ def iter_upload( timing between chunk retries or to supply a custom ``predicate`` for API-specific transient errors. Protocol recovery (such as server offset synchronization on missing status headers) is preserved automatically, - and terminal errors (such as ``DeadlineExceeded`` or - ``TransferStalledError``) are never retried. + and terminal errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. @@ -919,8 +939,9 @@ def resume( timing between chunk retries or to supply a custom ``predicate`` for API-specific transient errors. Protocol recovery (such as server offset synchronization on missing status headers) is preserved automatically, - and terminal errors (such as ``DeadlineExceeded`` or - ``TransferStalledError``) are never retried. + and terminal errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. @@ -970,8 +991,9 @@ def iter_resume( timing between chunk retries or to supply a custom ``predicate`` for API-specific transient errors. Protocol recovery (such as server offset synchronization on missing status headers) is preserved automatically, - and terminal errors (such as ``DeadlineExceeded`` or - ``TransferStalledError``) are never retried. + and terminal errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. diff --git a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py index cb046fccbfd1..3557b44f49c2 100644 --- a/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py +++ b/packages/google-api-core/google/api_core/resumable_transfer/upload_async.py @@ -175,8 +175,9 @@ def __init__( for the initial session creation request. Use this to customize exponential backoff timing (such as ``AsyncRetry(initial=1.0, maximum=60.0)``) or to supply a custom ``predicate`` function for API-specific transient - errors. Terminal errors (such as ``DeadlineExceeded`` or - ``TransferStalledError``) are never retried. + errors. Terminal errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. start_timeout: Optional timeout in seconds for the start request. """ self._config = config or ResumableUploadConfig() @@ -332,6 +333,9 @@ def _get_retry_predicate( if the error should be retried (from a user-supplied AsyncRetry instance). When provided, this function is evaluated for non-terminal errors while automatically preserving resumable upload state recovery. + Terminal errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. Returns: A callable accepting an exception and returning a boolean. @@ -375,7 +379,10 @@ def _get_async_retry( """Resolves unary AsyncRetry policy for start requests. Args: - retry_override: Optional unary AsyncRetry policy override for the start request. + retry_override: Optional unary AsyncRetry policy override for the start + request. Terminal errors (``DeadlineExceeded``, + ``TransferStalledError``, ``UploadCancelledError``, and + ``UnseekableStreamError``) are never retried. Returns: Configured or default unary AsyncRetry instance. @@ -404,6 +411,10 @@ def _get_async_streaming_retry( Args: retry_override: Optional retry policy override for chunk transmission. + Protocol recovery is preserved automatically, and terminal errors + (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. Returns: Configured or default AsyncStreamingRetry instance. @@ -446,7 +457,10 @@ async def initiate( content_type: Optional MIME type override of the payload. retry: Optional retry configuration (``AsyncRetry``) for this session initiation call. Overrides ``start_retry`` if provided. Use this to - customize backoff timing or add custom retryable exceptions. + customize backoff timing or add custom retryable exceptions. Terminal + errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-request timeout override in seconds. Returns: @@ -755,7 +769,11 @@ async def _transmit_all_chunks( computed_size: Total stream size in bytes, if known. progress_queue: Optional queue receiving UploadProgress snapshots. stream_obj: Underlying stream object for recovery seeking. - retry: Optional retry policy override for chunk transmission. + retry: Optional retry policy override for chunk transmission. Protocol + recovery is preserved automatically, and terminal errors + (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-attempt timeout ceiling in seconds. Returns: @@ -843,8 +861,9 @@ def upload( supply a custom ``predicate`` for API-specific transient errors. Protocol recovery (such as server offset synchronization on missing status headers) is preserved automatically, and terminal - errors (such as ``DeadlineExceeded`` or ``TransferStalledError``) - are never retried. + errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. @@ -933,8 +952,9 @@ def resume( supply a custom ``predicate`` for API-specific transient errors. Protocol recovery (such as server offset synchronization on missing status headers) is preserved automatically, and terminal - errors (such as ``DeadlineExceeded`` or ``TransferStalledError``) - are never retried. + errors (``DeadlineExceeded``, ``TransferStalledError``, + ``UploadCancelledError``, and ``UnseekableStreamError``) are never + retried. timeout: Optional per-attempt timeout ceiling in seconds. on_progress: Optional callback function receiving UploadProgress notifications. From f40173b31d71e2d81c7cc9d22db6d958507c9fc5 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 17 Sep 2026 15:26:15 +0000 Subject: [PATCH 42/43] attempt to resolve error failed to fetch commit --- .github/workflows/lint.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 702487b9d0fe..6f909b67b486 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -133,11 +133,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` - # See https://github.com/googleapis/google-cloud-python/issues/12013 - # and https://github.com/actions/checkout#checkout-head. with: - fetch-depth: 2 + fetch-depth: 0 persist-credentials: false - name: Setup Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 From 125388136f123589271bd504ed2e9f95a45bda8f Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 17 Sep 2026 15:27:18 +0000 Subject: [PATCH 43/43] Revert "attempt to resolve error failed to fetch commit" This reverts commit f40173b31d71e2d81c7cc9d22db6d958507c9fc5. --- .github/workflows/lint.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6f909b67b486..702487b9d0fe 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -133,8 +133,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. with: - fetch-depth: 0 + fetch-depth: 2 persist-credentials: false - name: Setup Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6