diff --git a/.gitignore b/.gitignore index 5ffdb6d8..1ca42da7 100644 --- a/.gitignore +++ b/.gitignore @@ -172,12 +172,3 @@ buildNumber.properties __pycache__/ .venv/ *.pyc -CONFORMITY_TEST_IMPROVEMENTS.md -S3_STORAGE_ANALYSIS.md -AZURE_BLOB_STORAGE_ANALYSIS.md -AZURE_BLOB_STORAGE_IMPROVEMENTS.md -SFTP_STORAGE_ANALYSIS.md -NFS_LOCKING.md -RELEASE-REVIEW.md -ASYNC_REVIEW.md -ASYNC_REVIEW_2.md diff --git a/AGENTS.md b/AGENTS.md index 89cbb4a9..fdaffa22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -195,6 +195,11 @@ To maintain a clear separation between fast, offline unit tests and containerize - Whenever acquiring an `UploadLock` (e.g. via `lockingService.lockUpload(uploadId)` or `lockingService.lockUploadByUri(uri)`), always manage the lock lifecycle using a `try-with-resources` block (e.g. `try (UploadLock lock = lockingService.lockUpload(id)) { ... }`). - This guarantees timely and reliable lock release upon normal completion or abrupt exceptions without requiring verbose and error-prone `try ... finally { if (lock != null) lock.release(); }` boilerplates. +### 25. UploadLock Cleanup & Non-Null UploadLockingService Invariant +- For all storage backends (Disk, S3, Azure), `terminateUpload` strictly manages storage resources (data, metadata, deduplication index) and does not perform lock cleanup. Active uploads are protected by an `UploadLock` whose close/release handle cleans up the active lock upon completion. +- Across all storage backends, `cleanupExpiredUploads()` explicitly calls `uploadLockingService.cleanupLock(uploadId)` to remove orphaned locks and stale index entries. +- `uploadLockingService` is guaranteed to never be null across storage backends (storage services initialize or fallback to default locking via `Utils.getLockingServiceOrDefault(uploadLockingService)`). Do NOT introduce redundant `if (uploadLockingService != null)` guard checks. + ## IETF Resumable Uploads for HTTP (RUFH) Spec Maintenance & Update Playbook diff --git a/CHANGELOG.md b/CHANGELOG.md index 90e46ee0..2ba77433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,13 +30,20 @@ See [docs/UPGRADING.md](docs/UPGRADING.md) for details and code examples. - **Azure Blob Storage**: `AzureBlobStorageService`, `AzureBlobLockingService` (Blob Leases with auto-renewal and cross-replica stop signals) and zero-copy `AzureBlobConcatenationService`. See [docs/AZURE_BLOB_STORAGE.md](docs/AZURE_BLOB_STORAGE.md). - **`UploadCompletionListener`**: register via `withUploadCompletionListener(...)` / `addUploadCompletionListener(...)`. Listeners run after the lock is released, so they can stream and delete the bytes immediately. Added `getUploadedBytes(UploadInfo)` and `deleteUpload(UploadInfo)` overloads. - **`withLockWaitTimeout(Duration)`** (default 60 s): maximum time a request waits for a contended upload lock; also derives the cloud chunk drain timeout (`lockWaitTimeout - 5 s`) and retry budget. See [docs/LOCKING.md](docs/LOCKING.md). -- **Automatic lock cleanup** on completion, termination (`DELETE`) and expiration across all locking services (`cleanupLock(...)`), plus a fallback sweeper for abandoned locks. +- **Automatic lock lifecycle cleanup** on release, termination (`DELETE`), and expiration across all locking services, deleting lock artifacts immediately without leaving lingering locks, plus a fallback sweeper for abandoned locks. - **JSON serialization**: store `UploadInfo` as JSON with `withJsonSerialization(true)`. - **Absolute base URLs**: `withUploadUri(String)` accepts e.g. `https://upload.example.com/files`, returning full `Location` URLs for Tus and RUFH; relative paths keep working. - **Multi-Backend Upload Deduplication & Lifecycle Cleanup**: Content-based upload deduplication with `withUploadDeduplication(true)` across Disk, S3, and Azure Blob storage. Features child reference markers, parent tombstoning, and complete index directory cleanup when uploads expire or are deleted. See [docs/DEDUPLICATION.md](docs/DEDUPLICATION.md). - **`NoOpUploadLockingService`**: Zero-overhead Null Object implementation of `UploadLockingService` for unit tests and custom environments, making locking service usage mandatory and eliminating defensive null checks across all storage backends, request handlers, and deduplication logic. ### Fixed +- **Azure Lock Cleanup**: eliminated lingering `.lock` blobs in Azure Blob Storage by having `AzureBlobUploadLock.release()` delete the actively leased blob directly using lease conditions upon lock release, and cleaning up orphaned lock artifacts in `cleanupExpiredUploads()`. +- **Storage Backend Utilities**: introduced dedicated `DiskUtils` and expanded `AzureUtils` alongside `S3Utils`, centralizing safe path resolution, storage directory initialization, path component validation, checksum normalization, blob existence checks, and quiet resource deletion across storage, checksum, and locking backends. +- **S3 Upload Performance**: reduced S3 upload latency by ~70% (~3x speedup in conformity benchmarks) via optimistic conditional writes (`If-None-Match: *`) during lock acquisition, caching part sizes to avoid repeated `statObject` roundtrips, and stream-based byte truncation. +- **Lock Lifecycle & Expired Upload Cleanup**: unified `cleanupExpiredUploads()` across Disk, S3, and Azure backends to clean up orphaned locks and prune stale checksum index entries, while ensuring `terminateUpload()` relies on the active `UploadLock` lifecycle. +- **Async Chunk Uploading**: hardened `AsyncChunkUploader` against servlet thread pool starvation under `CallerRunsPolicy` and fixed race conditions during temporary chunk file deletions. +- **Network Filesystem Locking**: prevented `FileChannel.lock()` `ENOLCK` failures on distributed filesystems (NFS, AWS EFS, Kubernetes PVs) by restricting kernel file locks exclusively to legacy `DiskLockingService`. +- **Deduplication Resilience**: protected deduplication traversal against infinite cyclic recursion (`StackOverflowError`) and handled missing concatenation parts safely. - **Content-Type matching**: media types are matched per RFC 9110 and tolerate parameters such as `;charset=UTF-8` (added by Spring `CharacterEncodingFilter`, proxies or clients) instead of returning `406`. - **Undertow & other containers**: `Content-Length` is cleared before `sendError(...)` ([#40](https://github.com/tomdesair/tus-java-server/issues/40)). - **Disk**: truncate underflow guard in `DiskStorageService`; `FileChannel` leak on lock acquisition errors fixed in `FileBasedLock`. diff --git a/docs/S3_STORAGE.md b/docs/S3_STORAGE.md index 4e6b3a28..539f6e55 100644 --- a/docs/S3_STORAGE.md +++ b/docs/S3_STORAGE.md @@ -272,11 +272,43 @@ To guarantee waterproof single-winner lock exclusivity across cloud providers an | **Wasabi** | Silently ignored (last-write-wins overwrite) | Layer 2 jitter arbitration or `withS3ConditionalWritesSupported(false)` | - On backends without conditional write support, contender nodes sleep for a brief randomized jitter (default: 20–60ms) to allow competing writes to settle, then re-read `GetObject`. - If the remote `holderId` does not match its own, the node detects that it was overtaken, rejects the acquisition, and leaves the winner's lock intact. -3. **Safe Expired Lock Eviction**: - - When evicting an expired lock, the lock object's expiration is verified again immediately before deletion to prevent evicting a fresh lock created by a winning peer. +3. **Safe Expired Lock Eviction with Sibling Mutex**: + - Evicting an expired lock requires a dedicated sibling mutex (`locks/.mutex`) using atomic conditional write (`If-None-Match: *`) to serialize eviction and prevent split-brain race conditions (see detailed explanation below). 4. **Owner-Safe Lock Release**: - In `S3UploadLock.close()`, the node verifies that the remote lock is still owned by its own `holderId` before deleting it. If its lease expired while the process was paused and another node took over ownership, the previous node will never delete the new owner's active lock. +### Why Was the Mutex Required in evictExpiredLock? (Preventing Split-Brain on Eviction) + +Unlike Azure Blob Storage (which supports conditional lease deletion via `x-ms-lease-id`), AWS S3 **does not support conditional `DELETE` operations** (such as `DeleteObject If-Match: `). S3's `DeleteObject` is always unconditional. + +This limitation introduces a critical concurrency hazard when multiple cluster nodes concurrently discover that an existing lock has expired: + +``` +Thread / Node A Thread / Node B +---------------- ---------------- +1. Reads expired lock L1. 1. Reads expired lock L1. +2. Evicts L1 (unconditional DELETE). +3. Writes new active lock L2 via + conditional PUT (If-None-Match: *). +4. Now holds active lock L2! + 2. Executes its delayed unconditional DELETE for L1, + BLOWING AWAY Thread A's active lock L2! + 3. Thread C arrives, sees no lock, and acquires L3. + ==> SPLIT-BRAIN: Both A and C believe they hold the lock! +``` + +#### How the Mutex Solves This +To eliminate this race condition, `S3LockingService` introduces an ephemeral **sibling mutex object** (`locks/.mutex`): +1. **Contender Mutex Election**: Before performing eviction, every contender attempts to conditionally create `locks/.mutex` using `If-None-Match: *`. +2. **Single Evictor**: Exactly one contender successfully writes the mutex. All other contenders fail with `412 Precondition Failed` and immediately back off with `false`, abandoning eviction. +3. **Protected Eviction**: Under the protection of the mutex, the winning node re-inspects the lock in S3. If it is still expired, the winner deletes the expired lock object and immediately releases the mutex. +4. **Self-Healing TTL**: If a node crashes mid-eviction while holding the mutex, the mutex contains a creation timestamp with a 5-second grace period (`MUTEX_GRACE_PERIOD_MS = 5000L`). Subsequent eviction attempts can automatically reclaim a stale mutex without deadlocks. + +#### Why `tryAcquireLock` Does NOT Require a Mutex +On the normal (uncontended or active) acquisition path, `tryAcquireLock` writes the lock lease directly using S3 atomic conditional `PUT` (`If-None-Match: *`). S3 natively guarantees that only one request creates the object, returning `HTTP 412 Precondition Failed` to all concurrent contenders. + +Because S3's conditional write provides atomic Compare-And-Swap (CAS) for `PUT`, **normal lock acquisition does not require any mutex**, preserving maximum throughput and single-roundtrip performance. The mutex is solely required to serialize the unconditional `DELETE` during expired lease eviction. + ### Jitter Configuration & Zero-Jitter Performance Tuning The jitter window can be customized via `.withJitter(minMs, maxMs)` on `S3LockingService`: diff --git a/scripts/rufh_conformity_test.py b/scripts/rufh_conformity_test.py index 89e53f0f..3cb05f07 100755 --- a/scripts/rufh_conformity_test.py +++ b/scripts/rufh_conformity_test.py @@ -1153,6 +1153,65 @@ def test_paused_upload_head_offset_and_resume(self, target_url, request): assert resp2.get(UPLOAD_OFFSET) == "92" assert resp2.get(UPLOAD_COMPLETE) == TRUE + def test_interrupted_upload_complete_patch_does_not_prematurely_complete(self, target_url, request): + """ + §4.2 & §5: Interrupted upload-completing PATCH must not prematurely finalize upload. + When a client sends a PATCH with Upload-Complete: ?1 and Content-Length: 100, but the + connection is prematurely closed after sending only 40 bytes, the server must persist the + partial 40 bytes, must NOT mark the upload as completed, and must leave the length + deferred so subsequent PATCH requests can resume and complete the upload. + """ + upload_uri = create_partial_upload(target_url, test_name=request.node.name) + + parsed = urlparse(upload_uri) + host = parsed.hostname or "localhost" + port = parsed.port or 80 + path = parsed.path + ("?" + parsed.query if parsed.query else "") + + s = socket.create_connection((host, port), timeout=10) + try: + req_lines = [ + f"PATCH {path} HTTP/1.1", + f"Host: {host}:{port}", + f"{UPLOAD_OFFSET}: 0", + f"{UPLOAD_COMPLETE}: {TRUE}", + f"{CONTENT_TYPE}: {APPLICATION_PARTIAL_UPLOAD}", + "Content-Length: 100", + "Connection: close", + "", + "" + ] + req_data = "\r\n".join(req_lines).encode("latin1") + s.sendall(req_data) + # Send only 40 bytes out of the 100 bytes declared in Content-Length + s.sendall(b"0123456789" * 4) + time.sleep(0.1) + # Abruptly close the socket to interrupt the stream + s.close() + except Exception: + pass + + # Give server a brief moment to commit partial data + time.sleep(0.5) + + # Check upload status via HEAD + status_h, h_headers, _, _ = http_request("HEAD", upload_uri, test_name=request.node.name) + assert status_h in (200, 204) + assert h_headers.get(UPLOAD_OFFSET) == "40", "Server must have saved the 40 received bytes" + assert h_headers.get(UPLOAD_COMPLETE) == FALSE, "Interrupted upload must NOT be marked complete" + + # Now resume from offset 40 and complete the remaining 60 bytes + status_resume, resp_resume, _, _ = http_request( + "PATCH", + upload_uri, + headers={UPLOAD_OFFSET: "40", UPLOAD_COMPLETE: TRUE, CONTENT_TYPE: APPLICATION_PARTIAL_UPLOAD}, + body=b"abcdefghij" * 6, # 60 bytes + test_name=request.node.name, + ) + assert status_resume in (200, 201, 204) + assert resp_resume.get(UPLOAD_OFFSET) == "100" + assert resp_resume.get(UPLOAD_COMPLETE) == TRUE + class TestHttpDigests: """Tests for HTTP Digests (RFC 9530).""" @@ -1267,6 +1326,45 @@ def make_abs(loc): assert get_body == payload http_request("DELETE", new_loc, test_name=request.node.name) + def test_content_digest_deduplication_active_child_preserved(self, target_url, request): + """ + RFC 9530 & RUFH: Deduplication Active Child Verification. + Uploads parent P and duplicate child C with identical Content-Digest. + Verifies both stream the exact payload. Preserves the active child and parent + in storage so that checksum index entries with children remain present for inspection. + """ + payload = b"Preserved active RUFH deduplicated child test payload: " + str(time.time()).encode("ascii") + sha256_b64 = base64.b64encode(hashlib.sha256(payload).digest()).decode("ascii") + headers = { + UPLOAD_COMPLETE: TRUE, + UPLOAD_LENGTH: str(len(payload)), + "Content-Digest": f"sha-256=:{sha256_b64}:", + "Repr-Digest": f"sha-256=:{sha256_b64}:", + CONTENT_TYPE: "text/plain", + } + + def make_abs(loc): + if not loc.startswith("http"): + parsed = urlparse(target_url) + return f"{parsed.scheme}://{parsed.netloc}{loc}" + return loc + + # 1. Create parent upload P + status1, resp1, _, _ = http_request("POST", target_url, headers=headers, body=payload, test_name=request.node.name) + assert status1 in (200, 201) + p_loc = make_abs(resp1.get(LOCATION)) + + # 2. Create child duplicate upload C + status2, resp2, _, _ = http_request("POST", target_url, headers=headers, body=payload, test_name=request.node.name) + assert status2 in (200, 201) + c_loc = make_abs(resp2.get(LOCATION)) + + # 3. Verify GET on P and C returns identical payload bytes + for loc in (p_loc, c_loc): + get_status, _, get_body, _ = http_request("GET", loc, test_name=request.node.name) + assert get_status == 200, f"GET {loc} must return 200, got {get_status}" + assert get_body == payload, "Child must stream full payload from shared parent data" + # CLI Entry Point & Custom Formatted Summary Reporter if __name__ == "__main__": diff --git a/scripts/tus_conformity_test.py b/scripts/tus_conformity_test.py index f3fdb4cf..64d7f9ad 100755 --- a/scripts/tus_conformity_test.py +++ b/scripts/tus_conformity_test.py @@ -995,6 +995,44 @@ def test_checksum_deduplication_and_parent_child_lifecycle(self, target_url): # Clean up http_request("DELETE", new_url, headers={TUS_RESUMABLE: TUS_API_VERSION}) + def test_checksum_deduplication_active_child_preserved(self, target_url): + """ + Deduplication Active Child Verification: + Uploads parent P and duplicate child C with identical SHA1 checksum. + Verifies both stream the exact payload. Preserves the active child and parent + in storage so that checksum index entries with children remain present for inspection. + """ + payload = b"Preserved active deduplicated child test payload: " + str(time.time()).encode("ascii") + digest_bytes = hashlib.sha1(payload).digest() + b64_digest = base64.b64encode(digest_bytes).decode("ascii") + checksum_header = f"sha1 {b64_digest}" + + # 1. Upload parent P + p_url, p_headers = create_upload( + target_url, + upload_length=str(len(payload)), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + extra_headers={UPLOAD_CHECKSUM: checksum_header}, + body=payload, + ) + assert p_headers.get(UPLOAD_OFFSET) == str(len(payload)) + + # 2. Upload duplicate child C + c_url, c_headers = create_upload( + target_url, + upload_length=str(len(payload)), + content_type=APPLICATION_OFFSET_OCTET_STREAM, + extra_headers={UPLOAD_CHECKSUM: checksum_header}, + body=payload, + ) + assert c_headers.get(UPLOAD_OFFSET) == str(len(payload)) + + # 3. Verify GET returns 200 with identical bytes for both parent and child + for loc in (p_url, c_url): + status, _, body = http_request("GET", loc, headers={TUS_RESUMABLE: TUS_API_VERSION}) + assert status == 200, f"GET {loc} must return 200, got {status}" + assert body == payload, "GET must return identical body bytes" + class TestTerminationExtension: """Termination Extension compliance tests (§7.5).""" diff --git a/src/main/java/me/desair/tus/server/TusFileUploadService.java b/src/main/java/me/desair/tus/server/TusFileUploadService.java index 76422bc6..2e2caa01 100644 --- a/src/main/java/me/desair/tus/server/TusFileUploadService.java +++ b/src/main/java/me/desair/tus/server/TusFileUploadService.java @@ -95,18 +95,20 @@ private static int calculateMaxLockRetries(Duration timeout) { } protected void initFeatures() { - // The order of the features is important + // The order of the features is important: ExpirationExtension must run before ChecksumExtension + // and HttpDigestsExtension so that upload expiration timestamps are already refreshed when + // deduplication triggers on upload completion. addTusExtension(new CoreProtocol()); CreationExtension creationExtension = new CreationExtension(); addTusExtension(creationExtension); addTusExtension(new CreationWithUploadExtension(creationExtension)); addTusExtension(new ResumableUploadsForHttpProtocol()); + addTusExtension(new ExpirationExtension()); addTusExtension(new ChecksumExtension()); + addTusExtension(new HttpDigestsExtension()); addTusExtension(new TerminationExtension()); - addTusExtension(new ExpirationExtension()); addTusExtension(new ConcatenationExtension()); addTusExtension(new CorsExtension()); - addTusExtension(new HttpDigestsExtension()); } /** @@ -606,7 +608,6 @@ public UploadInfo process( } if (wasInProgress && processedUploadInfo != null && !processedUploadInfo.isUploadInProgress()) { - cleanupLock(processedUploadInfo.getId()); notifyUploadCompletionListeners(processedUploadInfo); } @@ -626,7 +627,12 @@ protected UploadLock acquireUploadLock(HttpMethod method, String requestUri) break; } catch (TusException e) { if (HttpMethod.HEAD.equals(method) || HttpMethod.DELETE.equals(method)) { - uploadLockingService.requestLockRelease(requestUri); + // Throttled contention signal: write stop signal on initial contention (retries == 0) + // and every 12th iteration (~2400ms) rather than every 200ms to avoid write storms on + // S3/Azure. + if (retries == 0 || retries % 12 == 0) { + uploadLockingService.requestLockRelease(requestUri); + } retries++; try { Thread.sleep(LOCK_RETRY_INTERVAL_MS); @@ -635,6 +641,9 @@ protected UploadLock acquireUploadLock(HttpMethod method, String requestUri) throw new IOException("Lock acquisition retry interrupted", ie); } } else { + // Concurrent PATCH requests must fail fast (423/409) rather than waiting in a + // contention loop, because a resumed PATCH cannot know the actual committed offset + // without first issuing a HEAD request. throw e; } } @@ -974,6 +983,20 @@ protected UploadInfo processLockedRequest( throws IOException { ProtocolVersion detectedVersion = detectProtocolVersion(request); + // Record initial upload offset before request processing starts. + // If an error occurs (e.g. MaxAppendSizeExceededException), processTusException must only + // rollback bytes that were actually committed during this request, and never truncate bytes + // committed by earlier successful requests. + long initialOffset = 0L; + try { + UploadInfo preInfo = resolveUploadInfo(request, null, ownerKey); + if (preInfo != null) { + initialOffset = preInfo.getOffset(); + } + } catch (Exception e) { + log.debug("Could not resolve initial upload offset: {}", e.getMessage()); + } + try { validateRequest(method, request, ownerKey, detectedVersion); @@ -982,7 +1005,7 @@ protected UploadInfo processLockedRequest( return resolveUploadInfo(request, response, ownerKey); } catch (TusException e) { - processTusException(method, request, response, ownerKey, e, detectedVersion); + processTusException(method, request, response, ownerKey, e, detectedVersion, initialOffset); return null; } catch (RuntimeException e) { // Catch unexpected RuntimeException (e.g. NPE in custom extension) and convert it to an @@ -994,7 +1017,8 @@ protected UploadInfo processLockedRequest( new TusException( HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Internal server error: " + e.getMessage()); - processTusException(method, request, response, ownerKey, wrapped, detectedVersion); + processTusException( + method, request, response, ownerKey, wrapped, detectedVersion, initialOffset); return null; } } @@ -1046,16 +1070,6 @@ protected void notifyUploadCompletionListeners(UploadInfo uploadInfo) { } } - private void cleanupLock(UploadId id) { - if (id != null) { - try { - uploadLockingService.cleanupLock(id); - } catch (Exception e) { - log.debug("Error cleaning up lock for upload ID {}: {}", id, e.getMessage()); - } - } - } - public ProtocolVersion detectProtocolVersion(HttpServletRequest request) { return Utils.detectProtocolVersion(request, supportedProtocolVersion); } @@ -1104,6 +1118,18 @@ protected void processTusException( TusException exception, ProtocolVersion version) throws IOException { + processTusException(method, request, response, ownerKey, exception, version, 0L); + } + + protected void processTusException( + HttpMethod method, + TusServletRequest request, + TusServletResponse response, + String ownerKey, + TusException exception, + ProtocolVersion version, + long initialOffset) + throws IOException { int status = exception.getStatus(); String message = exception.getMessage(); @@ -1138,12 +1164,18 @@ protected void processTusException( } } - // Since an error occurred, the bytes we have written are probably not valid. - // So remove them. + // Since an error occurred, only rollback bytes that were actually committed during this + // request. Capping rollback to (uploadInfo.getOffset() - initialOffset) guarantees that if an + // uncommitted chunk failed (e.g. MaxAppendSizeExceededException where bytes were read into + // temp storage but never committed to offset), we do not truncate earlier valid data! String uploadUri = Utils.getUploadUri(request, response); UploadInfo uploadInfo = uploadStorageService.getUploadInfo(uploadUri, ownerKey); if (uploadInfo != null) { - uploadStorageService.removeLastNumberOfBytes(uploadInfo, request.getBytesRead()); + long committedThisRequest = Math.max(0L, uploadInfo.getOffset() - initialOffset); + long bytesToRollback = Math.min(request.getBytesRead(), committedThisRequest); + if (bytesToRollback > 0) { + uploadStorageService.removeLastNumberOfBytes(uploadInfo, bytesToRollback); + } } } catch (TusException ex) { diff --git a/src/main/java/me/desair/tus/server/checksum/ChecksumRequestHandler.java b/src/main/java/me/desair/tus/server/checksum/ChecksumRequestHandler.java index ae823fcd..0ff26a8b 100644 --- a/src/main/java/me/desair/tus/server/checksum/ChecksumRequestHandler.java +++ b/src/main/java/me/desair/tus/server/checksum/ChecksumRequestHandler.java @@ -64,20 +64,7 @@ public void process( if (uploadInfo != null && !uploadInfo.isUploadInProgress()) { uploadInfo.setChecksum(expectedValue); uploadInfo.setChecksumAlgorithm(checksumAlgorithm); - if (uploadStorageService.isUploadDeduplicationEnabled() - && uploadInfo.getDuplicatesUploadId() == null) { - try { - ChecksumKey key = ChecksumKey.of(expectedValue, checksumAlgorithm); - UploadInfo deduped = uploadStorageService.deduplicate(uploadInfo, key); - if (deduped != null) { - uploadInfo = deduped; - } - } catch (Exception e) { - uploadStorageService.update(uploadInfo); - } - } else { - uploadStorageService.update(uploadInfo); - } + uploadStorageService.update(uploadInfo); } } } diff --git a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java index ae253009..5459a000 100644 --- a/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/creationwithupload/CreationWithUploadPostRequestHandler.java @@ -14,6 +14,7 @@ import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; import me.desair.tus.server.util.Utils; +import org.apache.commons.lang3.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,7 +41,12 @@ public HttpProblemDetails process( throws IOException, TusException { Long contentLength = Utils.getLongHeader(servletRequest, HttpHeader.CONTENT_LENGTH); - if (contentLength != null && contentLength > 0) { + // Allow chunked transfer encoding where Content-Length is omitted or negative + boolean hasContent = + (contentLength != null && contentLength > 0) + || Strings.CI.contains( + servletRequest.getHeader(HttpHeader.TRANSFER_ENCODING), "chunked"); + if (hasContent) { String location = servletResponse.getHeader(HttpHeader.LOCATION); if (location != null) { UploadInfo uploadInfo = uploadStorageService.getUploadInfo(location, ownerKey); diff --git a/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java b/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java index 0fd677f0..1f214b83 100644 --- a/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/digest/HttpDigestsPostPutPatchRequestHandler.java @@ -9,7 +9,6 @@ import me.desair.tus.server.HttpMethod; import me.desair.tus.server.HttpProblemDetails; import me.desair.tus.server.checksum.ChecksumAlgorithm; -import me.desair.tus.server.checksum.ChecksumKey; import me.desair.tus.server.digest.validation.HttpDigestsValidator; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadDigestMismatchException; @@ -180,21 +179,7 @@ private void verifyReprDigestOnCompletion( // Always save verified checksum on the uploadInfo uploadInfo.setChecksum(expectedValue); uploadInfo.setChecksumAlgorithm(alg); - - if (uploadStorageService.isUploadDeduplicationEnabled() - && uploadInfo.getDuplicatesUploadId() == null) { - try { - ChecksumKey key = ChecksumKey.of(expectedValue, alg); - UploadInfo deduped = uploadStorageService.deduplicate(uploadInfo, key); - if (deduped != null) { - uploadInfo = deduped; - } - } catch (Exception e) { - uploadStorageService.update(uploadInfo); - } - } else { - uploadStorageService.update(uploadInfo); - } + uploadStorageService.update(uploadInfo); } } } diff --git a/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java b/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java index 6e8df0bc..aedc176d 100644 --- a/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java +++ b/src/main/java/me/desair/tus/server/expiration/ExpirationRequestHandler.java @@ -61,8 +61,16 @@ public HttpProblemDetails process( throw new UploadNotFoundException( "The upload for path " + uploadUri + " has expired and was removed."); } else { + Long previousExpiration = uploadInfo.getExpirationTimestamp(); uploadInfo.updateExpiration(expirationPeriod); - uploadStorageService.update(uploadInfo); + // Performance optimization: If create() or append() already refreshed the expiration + // timestamp when persisting metadata during this exact request (within a 1000ms window), + // skip calling update(uploadInfo) to avoid an expensive redundant remote S3/Azure PUT + // metadata round-trip. + if (previousExpiration == null + || Math.abs(uploadInfo.getExpirationTimestamp() - previousExpiration) > 1000L) { + uploadStorageService.update(uploadInfo); + } } } return null; diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java index f728a31e..90e13519 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandler.java @@ -59,13 +59,11 @@ public HttpProblemDetails process( String uploadCompleteHeader = servletRequest.getHeader(HttpHeader.UPLOAD_COMPLETE); Boolean uploadComplete = StructuredHeaderUtil.parseBoolean(uploadCompleteHeader); - // Per RUFH §4.2.1: If upload length was deferred and the client completes the upload - // via Upload-Complete: ?1, derive and set the total length from current offset + - // Content-Length. + boolean hadLengthInitially = uploadInfo.hasLength(); + long initialOffset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; long contentLength = servletRequest.getContentLengthLong(); - if (Boolean.TRUE.equals(uploadComplete) && !uploadInfo.hasLength() && contentLength >= 0) { - long currentOffset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; - uploadInfo.setLength(currentOffset + contentLength); + if (Boolean.TRUE.equals(uploadComplete) && !hadLengthInitially && contentLength >= 0) { + uploadInfo.setLength(initialOffset + contentLength); uploadStorageService.update(uploadInfo); } @@ -103,10 +101,26 @@ public HttpProblemDetails process( } } - boolean isFinished = Boolean.TRUE.equals(uploadComplete) || isUploadCompleted(uploadInfo); + // Detect whether stream was prematurely interrupted (e.g. concurrent contention or socket drop) + // or fewer bytes arrived than expected by Content-Length. + boolean streamInterrupted = + (interruptibleStream != null && interruptibleStream.isInterrupted()) + || (contentLength >= 0 && (uploadInfo.getOffset() - initialOffset) < contentLength); + + // Per RUFH §4.2: An upload must only be marked complete if the stream completed uninterrupted. + // If the stream was interrupted, we acknowledge partial progress and allow subsequent PATCH + // resumes. + boolean isFinished = + !streamInterrupted + && (Boolean.TRUE.equals(uploadComplete) || isUploadCompleted(uploadInfo)); if (isFinished) { uploadInfo.setLength(uploadInfo.getOffset()); uploadStorageService.update(uploadInfo); + } else if (streamInterrupted && !hadLengthInitially) { + // If the stream was interrupted on an upload whose length was deferred, + // reset length to null so that it remains open for subsequent resumes. + uploadInfo.setLength(null); + uploadStorageService.update(uploadInfo); } servletResponse.setHeader(HttpHeader.UPLOAD_OFFSET, String.valueOf(uploadInfo.getOffset())); diff --git a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java index df16b37f..991bc758 100644 --- a/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java +++ b/src/main/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandler.java @@ -147,8 +147,21 @@ public HttpProblemDetails process( } } - boolean isFinished = Boolean.TRUE.equals(uploadComplete) || isUploadCompleted(uploadInfo); - if (isFinished) { + // Detect whether stream was prematurely interrupted (e.g. concurrent contention or socket drop) + // or fewer bytes arrived than expected by Content-Length. + boolean streamInterrupted = + (interruptibleStream != null && interruptibleStream.isInterrupted()) + || (contentLength >= 0 && uploadInfo != null && uploadInfo.getOffset() < contentLength); + + // Per RUFH §4.2: An upload must only be marked complete if the stream completed uninterrupted. + // If the stream was interrupted, we acknowledge partial progress and allow subsequent PATCH + // resumes. + boolean isFinished = + !streamInterrupted + && (Boolean.TRUE.equals(uploadComplete) + || (uploadInfo != null && isUploadCompleted(uploadInfo))); + + if (isFinished && uploadInfo != null) { uploadInfo.setLength(uploadInfo.getOffset()); uploadStorageService.update(uploadInfo); servletResponse.setStatus(200); @@ -157,11 +170,19 @@ public HttpProblemDetails process( HttpHeader.UPLOAD_COMPLETE, StructuredHeaderUtil.formatBoolean(true)); servletResponse.setHeader(HttpHeader.UPLOAD_OFFSET, String.valueOf(uploadInfo.getOffset())); } else { + // If length was originally deferred (no Upload-Length header) and stream was interrupted, + // reset length to null so that the upload remains in-progress and can be resumed via PATCH. + if (streamInterrupted && uploadInfo != null && uploadLength == null) { + uploadInfo.setLength(null); + uploadStorageService.update(uploadInfo); + } servletResponse.setStatus(201); servletResponse.setHeader(HttpHeader.LOCATION, uploadUri); servletResponse.setHeader( HttpHeader.UPLOAD_COMPLETE, StructuredHeaderUtil.formatBoolean(false)); - servletResponse.setHeader(HttpHeader.UPLOAD_OFFSET, String.valueOf(uploadInfo.getOffset())); + servletResponse.setHeader( + HttpHeader.UPLOAD_OFFSET, + String.valueOf(uploadInfo != null ? uploadInfo.getOffset() : 0L)); } return null; } diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java index ee24adfe..89b38c08 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLock.java @@ -4,6 +4,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import me.desair.tus.server.util.Utils; import org.slf4j.Logger; @@ -15,7 +16,7 @@ *

Provides common lifecycle management for TTL-based locks including: * *

    - *
  • Background heartbeat lease auto-renewal daemon scheduling. + *
  • Background heartbeat lease auto-renewal task execution via a shared service executor. *
  • Active request input stream registration tracking and clean shutdown. *
*/ @@ -24,49 +25,36 @@ public abstract class AbstractLeaseLock implements UploadLock { private static final Logger log = LoggerFactory.getLogger(AbstractLeaseLock.class); private final LeaseData leaseData; - private ScheduledExecutorService heartbeatExecutor; - private final Map activeInputStreams; + private final UploadId uploadId; + private final Map activeInputStreams; + private final ScheduledFuture heartbeatTask; /** - * Constructs an active lock and schedules the background heartbeat renewal daemon. + * Constructs an active lock and schedules the background heartbeat renewal task using the shared + * service-level executor. * * @param leaseData The lease metadata + * @param uploadId The upload identifier associated with this lock * @param activeInputStreams Map of active request input streams in the JVM - * @param watchdogThreadName Name prefix for the watchdog heartbeat thread - */ - protected AbstractLeaseLock( - LeaseData leaseData, Map activeInputStreams, String watchdogThreadName) { - this(leaseData, activeInputStreams, null, watchdogThreadName); - } - - /** - * Full constructor allowing injection of a custom executor (e.g. for testing). - * - * @param leaseData The lease metadata - * @param activeInputStreams Map of active request input streams in the JVM - * @param heartbeatExecutor ScheduledExecutorService for lease renewal, or null to auto-schedule - * @param watchdogThreadName Name prefix for the watchdog heartbeat thread + * @param heartbeatExecutor Shared ScheduledExecutorService for lease renewal */ protected AbstractLeaseLock( LeaseData leaseData, - Map activeInputStreams, - ScheduledExecutorService heartbeatExecutor, - String watchdogThreadName) { + UploadId uploadId, + Map activeInputStreams, + ScheduledExecutorService heartbeatExecutor) { this.leaseData = Objects.requireNonNull(leaseData, "leaseData must not be null"); + this.uploadId = uploadId; this.activeInputStreams = activeInputStreams; long leaseDurationMs = leaseData.getLeaseDurationMs(); - if (heartbeatExecutor != null) { - this.heartbeatExecutor = heartbeatExecutor; - } else if (leaseDurationMs > 0 && watchdogThreadName != null) { + if (heartbeatExecutor != null && leaseDurationMs > 0) { long renewalPeriodMs = Math.max(1000L, leaseDurationMs / 3); - this.heartbeatExecutor = - Utils.scheduleWatchdog( - watchdogThreadName + "-" + leaseData.getHolderId(), - this::renewLease, - renewalPeriodMs, - renewalPeriodMs, - TimeUnit.MILLISECONDS); + this.heartbeatTask = + heartbeatExecutor.scheduleWithFixedDelay( + this::renewLease, renewalPeriodMs, renewalPeriodMs, TimeUnit.MILLISECONDS); + } else { + this.heartbeatTask = null; } } @@ -82,6 +70,10 @@ public String getRequestUri() { return leaseData.getRequestUri(); } + public UploadId getUploadId() { + return uploadId; + } + public long getLeaseDurationMs() { return leaseData.getLeaseDurationMs(); } @@ -110,12 +102,14 @@ public void release() { @Override public void close() { - // 1. Terminate heartbeat daemon cleanly via centralized Utils helper - Utils.shutdownExecutor(heartbeatExecutor); + // 1. Cancel heartbeat task cleanly + if (heartbeatTask != null) { + heartbeatTask.cancel(true); + } // 2. Remove active stream registration from JVM heap - if (activeInputStreams != null && getRequestUri() != null) { - activeInputStreams.remove(getRequestUri()); + if (activeInputStreams != null && uploadId != null) { + activeInputStreams.remove(uploadId); } // 3. Release backend storage resources @@ -135,40 +129,40 @@ public void renewLease() { leaseData.setExpiresAt(oldExpiresAt); // Ownership lost (e.g. lease taken over by another node): abort immediately log.info( - "Aborting active input stream for upload URI {} as lock was taken over: {}", - getRequestUri(), + "Aborting active input stream for upload ID {} as lock was taken over: {}", + uploadId, e.getMessage()); abortActiveStreamAndHeartbeat(); } catch (Exception e) { leaseData.setExpiresAt(oldExpiresAt); - // Transient failure (e.g. network timeout). Only abort if the previous lease actually - // expired. + // Transient failure (e.g. network timeout). Only abort if previous lease actually expired. if (System.currentTimeMillis() >= oldExpiresAt) { log.warn( - "Lease for upload URI {} expired after renewal failure; aborting active stream", - getRequestUri(), + "Lease for upload ID {} expired after renewal failure; aborting active stream", + uploadId, e); abortActiveStreamAndHeartbeat(); } else { log.warn( - "Transient failure renewing lease for upload URI {} (will retry on next tick): {}", - getRequestUri(), + "Transient failure renewing lease for upload ID {} (will retry on next tick): {}", + uploadId, e.getMessage()); } } } private void abortActiveStreamAndHeartbeat() { - if (activeInputStreams != null && getRequestUri() != null) { - InputStream stream = activeInputStreams.get(getRequestUri()); + if (activeInputStreams != null && uploadId != null) { + InputStream stream = activeInputStreams.remove(uploadId); if (stream != null) { log.info( - "Aborting active input stream for upload URI {} due to lease renewal failure", - getRequestUri()); + "Aborting active input stream for upload ID {} due to lease renewal failure", uploadId); Utils.interruptStream(stream); } } - Utils.shutdownExecutor(heartbeatExecutor); + if (heartbeatTask != null) { + heartbeatTask.cancel(true); + } } /** diff --git a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java index ac14ac8e..92eacb88 100644 --- a/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/AbstractLeaseLockingService.java @@ -46,7 +46,7 @@ public abstract class AbstractLeaseLockingService extends AbstractCloseableResou protected long jitterMinMs = DEFAULT_JITTER_MIN_MS; protected long jitterMaxMs = DEFAULT_JITTER_MAX_MS; - protected final Map activeInputStreams = new ConcurrentHashMap<>(); + protected final Map activeInputStreams = new ConcurrentHashMap<>(); protected final ScheduledExecutorService watchdogExecutor; /** @@ -85,15 +85,42 @@ protected AbstractLeaseLockingService( this.leaseDurationMs = leaseDurationMs; this.pollIntervalMs = pollIntervalMs; + // Use a multi-threaded daemon scheduled executor (scaling from 2 to 8 threads based on + // available processors) so that stop signal polling and lease heartbeats across concurrent + // uploads do not suffer head-of-line blocking during cloud network I/O. + int poolSize = Math.max(2, Math.min(Runtime.getRuntime().availableProcessors(), 8)); this.watchdogExecutor = Utils.scheduleWatchdog( watchdogThreadName, + poolSize, this::checkStopSignals, pollIntervalMs, pollIntervalMs, TimeUnit.MILLISECONDS); } + /** + * Constructor allowing injection of a custom or null watchdog executor (e.g. for testing). + * + * @param idFactory The factory used to parse and generate upload identifiers + * @param leaseDurationMs Lock lease duration in milliseconds + * @param pollIntervalMs Watchdog polling interval in milliseconds + * @param shutdownHookName Name of the JVM shutdown hook thread, or null if none + * @param watchdogExecutor Custom executor service, or null + */ + protected AbstractLeaseLockingService( + UploadIdFactory idFactory, + long leaseDurationMs, + long pollIntervalMs, + String shutdownHookName, + ScheduledExecutorService watchdogExecutor) { + super(shutdownHookName); + this.idFactory = Objects.requireNonNull(idFactory, "The idFactory cannot be null"); + this.leaseDurationMs = leaseDurationMs; + this.pollIntervalMs = pollIntervalMs; + this.watchdogExecutor = watchdogExecutor; + } + @Override public UploadLock lockUploadByUri(String requestUri) throws TusException, IOException { UploadId uploadId = idFactory.readUploadId(requestUri); @@ -140,7 +167,10 @@ public void setIdFactory(UploadIdFactory idFactory) { @Override public void registerInputStream(String requestUri, InputStream inputStream) { if (requestUri != null && inputStream != null) { - activeInputStreams.put(requestUri, inputStream); + UploadId uploadId = idFactory.readUploadId(requestUri); + if (uploadId != null) { + activeInputStreams.put(uploadId, inputStream); + } } } @@ -150,19 +180,42 @@ public void requestLockRelease(String requestUri) { return; } - // 1. Interrupt active local input stream in this JVM - InputStream activeStream = activeInputStreams.remove(requestUri); - if (activeStream != null) { - Utils.interruptStream(activeStream); - } - - // 2. Write a .stop signal to notify remote cluster replicas / pods UploadId uploadId = idFactory.readUploadId(requestUri); if (uploadId != null) { + // 1. Interrupt active local input stream in this JVM directly by UploadId + InputStream activeStream = activeInputStreams.remove(uploadId); + if (activeStream != null) { + Utils.interruptStream(activeStream); + } + + // 2. Write a .stop signal to notify remote cluster replicas / pods writeStopSignal(uploadId); } } + @Override + public void cleanupLock(UploadId id) throws IOException { + if (id != null) { + clearStopSignal(id); + } + } + + /** + * Clears any lingering contention stop signal for the given upload ID. + * + * @param uploadId The upload identifier + */ + protected void clearStopSignal(UploadId uploadId) {} + + /** + * Returns the shared scheduled executor used for background watchdog polling. + * + * @return The scheduled executor service + */ + public ScheduledExecutorService getWatchdogExecutor() { + return watchdogExecutor; + } + @Override protected void cleanupOnClose() throws IOException { Utils.shutdownExecutor(watchdogExecutor); @@ -198,7 +251,7 @@ protected UploadLock acquireOrEvictExpiredLock( LeaseData leaseData = new LeaseData(holderId, requestUri, leaseDurationMs, expiresAt, now, null, null); - UploadLock lock = tryAcquireLock(uploadId, leaseData); + UploadLock lock = tryAcquireLock(uploadId, leaseData, false); if (lock != null) { return lock; } @@ -211,7 +264,7 @@ protected UploadLock acquireOrEvictExpiredLock( now = System.currentTimeMillis(); leaseData.setAcquiredAt(now); leaseData.setExpiresAt(now + leaseDurationMs); - lock = tryAcquireLock(uploadId, leaseData); + lock = tryAcquireLock(uploadId, leaseData, true); if (lock != null) { return lock; } @@ -220,6 +273,22 @@ protected UploadLock acquireOrEvictExpiredLock( return null; } + /** + * Subclass implementation of primary lock acquisition, indicating whether this call is an + * eviction retry after deleting an expired lock. Default implementation delegates to {@link + * #tryAcquireLock(UploadId, LeaseData)}. + * + * @param uploadId The upload identifier + * @param leaseData The lease metadata describing the lock to acquire + * @param isEvictionRetry True if this acquisition follows an expired lock eviction + * @return Acquired {@link UploadLock}, or null if already held + * @throws IOException If an I/O error occurs + */ + protected UploadLock tryAcquireLock( + UploadId uploadId, LeaseData leaseData, boolean isEvictionRetry) throws IOException { + return tryAcquireLock(uploadId, leaseData); + } + /** * Subclass implementation of atomic primary lock acquisition. * @@ -255,17 +324,17 @@ protected abstract UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseD protected abstract void writeStopSignal(UploadId uploadId); /** - * Checks for a contention interrupt stop signal for an active input stream entry. + * Checks for a contention interrupt stop signal for an active input stream. * - * @param uri The upload request URI + * @param uploadId The upload identifier * @param inputStream The active request input stream */ - protected abstract void checkStopSignalForEntry(String uri, InputStream inputStream); + protected abstract void checkStopSignalForUpload(UploadId uploadId, InputStream inputStream); /** Polls active streams against stop signals across replicas. */ public void checkStopSignals() { - for (Map.Entry entry : activeInputStreams.entrySet()) { - checkStopSignalForEntry(entry.getKey(), entry.getValue()); + for (Map.Entry entry : activeInputStreams.entrySet()) { + checkStopSignalForUpload(entry.getKey(), entry.getValue()); } } @@ -337,9 +406,4 @@ public void setJitter(long minMs, long maxMs) { this.jitterMinMs = minMs; this.jitterMaxMs = maxMs; } - - @Override - public void cleanupLock(String uploadUri) throws IOException { - cleanupLock(uploadUri != null && idFactory != null ? idFactory.readUploadId(uploadUri) : null); - } } diff --git a/src/main/java/me/desair/tus/server/upload/NoOpUploadLockingService.java b/src/main/java/me/desair/tus/server/upload/NoOpUploadLockingService.java index 6ba4cd02..92027797 100644 --- a/src/main/java/me/desair/tus/server/upload/NoOpUploadLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/NoOpUploadLockingService.java @@ -47,11 +47,6 @@ public boolean isLocked(UploadId id) { return false; } - @Override - public void cleanupLock(UploadId id) throws IOException { - // No-op - } - @Override public void registerInputStream(String requestUri, InputStream inputStream) { // No-op diff --git a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java index 8929f432..f730126e 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadLockingService.java @@ -90,28 +90,16 @@ default void close() throws IOException { } /** - * Cleans up the lock file or resources associated with a specific completed, terminated, or - * expired upload identifier. + * Cleans up lock artifacts and stop signals associated with the given upload. * - *

This method is invoked automatically when an upload completes successfully, is terminated - * via DELETE, or is cleaned up during expiration sweeps, freeing underlying lock storage (such as - * Azure Blob Lease lock blobs or on-disk lock files). + *

Called upon upload termination (e.g. DELETE) or expiration cleanup so that locking + * implementations can delete backend-specific lock files, blobs, or contention signals without + * storage services needing awareness of locking internals. * - * @param id The upload identifier of the completed, terminated, or expired upload - * @throws IOException If cleaning up lock resources fails + * @param id The ID of the upload to clean up + * @throws IOException If cleanup fails */ default void cleanupLock(UploadId id) throws IOException { // No-op by default for backwards compatibility } - - /** - * Cleans up the lock file or resources associated with a specific completed, terminated, or - * expired upload URI. - * - * @param uploadUri The request URI of the completed, terminated, or expired upload - * @throws IOException If cleaning up lock resources fails - */ - default void cleanupLock(String uploadUri) throws IOException { - // No-op by default for backwards compatibility - } } diff --git a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java index 88acbe4d..404ec29b 100644 --- a/src/main/java/me/desair/tus/server/upload/UploadStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/UploadStorageService.java @@ -262,9 +262,6 @@ void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) *

  • Delete the metadata record ({@code .info}) for this upload. *
  • If upload deduplication is enabled, coordinate index removal or tombstoning via {@link * me.desair.tus.server.upload.checksum.DeduplicationService#removeUploadFromIndex(UploadInfo)}. - *
  • Clean up any associated upload lock or contention artifacts (e.g. by delegating to {@link - * UploadLockingService#cleanupLock(UploadId)}) to ensure no orphaned lock files, stop - * signals, or lease objects remain in the locking layer. * * * @param uploadInfo The upload to terminate diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobChecksumIndex.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobChecksumIndex.java index 358ef2b1..df60b4d6 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobChecksumIndex.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobChecksumIndex.java @@ -62,7 +62,7 @@ public AzureBlobChecksumIndex(BlobContainerClient containerClient) { public AzureBlobChecksumIndex(BlobContainerClient containerClient, String checksumsPrefix) { this.containerClient = Objects.requireNonNull(containerClient, "containerClient must not be null"); - this.checksumsPrefix = sanitizePrefix(checksumsPrefix); + this.checksumsPrefix = AzureUtils.sanitizePrefix(checksumsPrefix); } @Override @@ -123,7 +123,7 @@ public void markParentDeleted(ChecksumKey key) throws IOException { @Override public boolean isParentDeleted(ChecksumKey key) throws IOException { Objects.requireNonNull(key, "key must not be null"); - return blobExists(buildKey(key, PARENT_DELETED_OBJECT)); + return AzureUtils.blobExists(containerClient, buildKey(key, PARENT_DELETED_OBJECT)); } @Override @@ -140,8 +140,7 @@ public void removeChild(ChecksumKey key, UploadId childId) throws IOException { Objects.requireNonNull(key, "key must not be null"); Objects.requireNonNull(childId, "childId must not be null"); String childKey = buildKey(key, CHILDREN_PREFIX + childId); - BlobClient childBlob = containerClient.getBlobClient(childKey); - childBlob.deleteIfExists(); + AzureUtils.deleteBlobQuietly(containerClient, childKey); } @Override @@ -165,7 +164,7 @@ public void deleteIndex(ChecksumKey key) throws IOException { try { ListBlobsOptions options = new ListBlobsOptions().setPrefix(prefix); for (BlobItem item : containerClient.listBlobs(options, null)) { - containerClient.getBlobClient(item.getName()).deleteIfExists(); + AzureUtils.deleteBlobQuietly(containerClient, item.getName()); } } catch (Exception e) { throw new IOException( @@ -210,26 +209,4 @@ public Set getAllIndexedKeys() throws IOException { private String buildKey(ChecksumKey key, String subPath) { return checksumsPrefix + key + (subPath.isEmpty() ? "" : "/" + subPath); } - - private boolean blobExists(String blobKey) { - try { - return Boolean.TRUE.equals(containerClient.getBlobClient(blobKey).exists()); - } catch (Exception e) { - return false; - } - } - - private static String sanitizePrefix(String prefix) { - if (prefix == null || prefix.trim().isEmpty()) { - return ""; - } - String trimmed = prefix.trim(); - while (trimmed.startsWith("/")) { - trimmed = trimmed.substring(1); - } - if (!trimmed.isEmpty() && !trimmed.endsWith("/")) { - trimmed = trimmed + "/"; - } - return trimmed; - } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java index 697f9be1..ff77e589 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationService.java @@ -74,7 +74,7 @@ public AzureBlobConcatenationService( UploadStorageService storageService) { this.containerClient = Objects.requireNonNull(containerClient, "containerClient must not be null"); - this.uploadPrefix = sanitizePrefix(uploadPrefix); + this.uploadPrefix = AzureUtils.sanitizePrefix(uploadPrefix); this.storageService = Objects.requireNonNull(storageService, "storageService must not be null"); } @@ -109,7 +109,8 @@ public void merge(UploadInfo finalUpload) throws IOException, UploadNotFoundExce int sequence = 0; for (UploadInfo partialInfo : partialUploads) { String blockId = generateBlockId(sequence++); - BlobClient partialBlob = containerClient.getBlobClient(uploadPrefix + partialInfo.getId()); + String partialBlobName = resolvePartialBlobName(partialInfo); + BlobClient partialBlob = containerClient.getBlobClient(partialBlobName); try { // 1. Attempt zero-copy server-side block copying on Azure Storage cluster using an @@ -226,14 +227,6 @@ private String generateBlockId(int index) { return Base64.getEncoder().encodeToString(idString.getBytes(StandardCharsets.UTF_8)); } - private String sanitizePrefix(String prefix) { - if (prefix == null || prefix.isEmpty()) { - return ""; - } - String result = prefix.startsWith("/") ? prefix.substring(1) : prefix; - return result.endsWith("/") ? result : result + "/"; - } - void stageBlockFromUrl(BlockBlobClient finalBlockBlob, String blockId, String sourceUrl) { finalBlockBlob.stageBlockFromUrl(blockId, sourceUrl, null); } @@ -265,4 +258,22 @@ String getAuthorizedBlobUrl(BlobClient blobClient) { } return blobClient.getBlobUrl(); } + + /** + * Resolves the physical Azure Blob name for a partial upload, resolving duplicate child uploads + * dynamically to their parent blob to prevent stitching against non-existent child blob targets. + * + * @param partialInfo The partial upload info + * @return The physical blob name + */ + String resolvePartialBlobName(UploadInfo partialInfo) { + if (storageService instanceof AzureBlobStorageService) { + return ((AzureBlobStorageService) storageService).getAzureBlobName(partialInfo); + } else if (partialInfo != null && partialInfo.getDuplicatesUploadId() != null) { + return uploadPrefix + partialInfo.getDuplicatesUploadId(); + } else if (partialInfo != null) { + return uploadPrefix + partialInfo.getId(); + } + return uploadPrefix; + } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java index 0bebcb34..e57b3362 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobLockingService.java @@ -98,12 +98,30 @@ public AzureBlobLockingService(BlobContainerClient containerClient, String locks super("azure-lock-shutdown-hook"); this.containerClient = Objects.requireNonNull(containerClient, "containerClient must not be null"); - this.locksPrefix = sanitizePrefix(locksPrefix); + this.locksPrefix = AzureUtils.sanitizePrefix(locksPrefix); - // Use pooled scheduled daemon executor to poll .stop signals across pods without thread leaks + // Use multi-threaded pooled scheduled daemon executor to poll .stop signals across pods + // and execute heartbeat renewals concurrently without head-of-line blocking + int poolSize = Math.max(2, Math.min(Runtime.getRuntime().availableProcessors(), 8)); this.watchdogExecutor = Utils.scheduleWatchdog( - "azure-lock-watchdog", this::pollStopSignals, 2000L, 2000L, TimeUnit.MILLISECONDS); + "azure-lock-watchdog", + poolSize, + this::pollStopSignals, + 2000L, + 2000L, + TimeUnit.MILLISECONDS); + } + + AzureBlobLockingService( + BlobContainerClient containerClient, + String locksPrefix, + ScheduledExecutorService watchdogExecutor) { + super("azure-lock-shutdown-hook"); + this.containerClient = + Objects.requireNonNull(containerClient, "containerClient must not be null"); + this.locksPrefix = AzureUtils.sanitizePrefix(locksPrefix); + this.watchdogExecutor = watchdogExecutor; } @Override @@ -145,7 +163,7 @@ private UploadLock lockUploadInternal(UploadId uploadId, String uriOrId) String idStr = uploadId.toString(); // 1. Target lock blob and instantiate Azure Blob Lease client - BlobClient lockBlob = containerClient.getBlobClient(locksPrefix + idStr + ".lock"); + BlobClient lockBlob = getBlobClient(locksPrefix + idStr + ".lock"); BlobLeaseClient leaseClient = createBlobLeaseClient(lockBlob); try { @@ -169,9 +187,17 @@ private UploadLock lockUploadInternal(UploadId uploadId, String uriOrId) // Lock successfully acquired: clear any lingering .stop signal blob deleteStopSignalBlob(idStr); + BlobClient stopBlob = getBlobClient(locksPrefix + idStr + ".stop"); AzureBlobUploadLock lock = new AzureBlobUploadLock( - leaseClient, lockBlob, uriOrId, idStr, activeStreams, activeLocks); + leaseClient, + lockBlob, + stopBlob, + uriOrId, + watchdogExecutor, + idStr, + activeStreams, + activeLocks); activeLocks.put(idStr, lock); return lock; } catch (BlobStorageException e) { @@ -186,37 +212,6 @@ private UploadLock lockUploadInternal(UploadId uploadId, String uriOrId) } } - @Override - public void cleanupLock(UploadId id) throws IOException { - if (id == null) { - return; - } - String idStr = id.toString(); - try { - // 1. Delete any lingering cross-pod .stop signal blob from prior lock contention - deleteStopSignalBlob(idStr); - - // 2. Delete the target .lock blob in the locks prefix - BlobClient lockBlob = getBlobClient(locksPrefix + idStr + ".lock"); - AzureBlobUploadLock activeLock = activeLocks.get(idStr); - if (activeLock != null && activeLock.getLeaseClient() != null) { - // The lock is actively held by this instance (e.g. during DELETE termination). - // Safely delete the blob specifying the active lease ID condition. - deleteLockBlobWithLease(lockBlob, activeLock.getLeaseClient().getLeaseId()); - activeLock.markBlobDeleted(); - } else { - deleteLockBlobIfUnleased(lockBlob); - } - } catch (Exception e) { - log.debug("Failed to clean up Azure lock blob for upload ID {}: {}", idStr, e.getMessage()); - } - } - - @Override - public void cleanupLock(String uploadUri) throws IOException { - cleanupLock(uploadUri != null && idFactory != null ? idFactory.readUploadId(uploadUri) : null); - } - /** * Safely deletes an actively leased lock blob using its active lease ID. * @@ -275,6 +270,17 @@ boolean deleteLockBlobIfUnleased(BlobClient lockBlob) { return false; } + @Override + public void cleanupLock(UploadId id) { + if (id == null) { + return; + } + String idStr = id.toString(); + deleteStopSignalBlob(idStr); + BlobClient lockBlob = getBlobClient(locksPrefix + idStr + ".lock"); + deleteLockBlobIfUnleased(lockBlob); + } + @Override public void cleanupStaleLocks() throws IOException { try { @@ -305,7 +311,7 @@ public void cleanupStaleLocks() throws IOException { if (blobName.endsWith(".stop")) { if (lastModified != null && lastModified.isBefore(now.minusSeconds(STOP_SIGNAL_EXPIRATION_SECONDS))) { - getBlobClient(blobName).deleteIfExists(); + AzureUtils.deleteBlobQuietly(getBlobClient(blobName)); log.info("Cleaned up stale Azure stop signal blob: {}", blobName); } } @@ -336,7 +342,7 @@ public boolean isLocked(UploadId id) { if (id == null) { return false; } - BlobClient lockBlob = containerClient.getBlobClient(locksPrefix + id + ".lock"); + BlobClient lockBlob = getBlobClient(locksPrefix + id + ".lock"); try { // Single HEAD call to fetch properties and check if LeaseState is "leased" BlobProperties props = lockBlob.getProperties(); @@ -380,7 +386,7 @@ private void interruptLocalStream(String idStr) { /** Creates a .stop signal blob to request remote pods to halt active streaming appends. */ void createStopSignalBlob(String idStr) { try { - BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); + BlobClient stopBlob = getBlobClient(locksPrefix + idStr + ".stop"); stopBlob.upload(BinaryData.fromString("stop"), true); } catch (Exception e) { log.debug("Failed to write .stop signal blob for upload ID {}: {}", idStr, e.getMessage()); @@ -389,20 +395,16 @@ void createStopSignalBlob(String idStr) { /** Deletes the .stop signal blob after lock acquisition. */ void deleteStopSignalBlob(String idStr) { - try { - BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); - stopBlob.deleteIfExists(); - } catch (Exception ignored) { - // Ignore cleanup exceptions - } + AzureUtils.deleteBlobQuietly(getBlobClient(locksPrefix + idStr + ".stop")); } /** Ensures the lock target blob exists on Azure Blob Storage. */ void ensureLockBlobExists(BlobClient lockBlob) { try { - if (!lockBlob.exists()) { - lockBlob.upload(BinaryData.fromBytes("lock".getBytes(StandardCharsets.UTF_8)), false); - } + // Calling upload(..., false) directly without lockBlob.exists() saves an expensive HEAD + // call. If another concurrent thread or pod already created the blob, Azure returns 409 + // Conflict or PreconditionFailed, which is caught and ignored safely. + lockBlob.upload(BinaryData.fromBytes("lock".getBytes(StandardCharsets.UTF_8)), false); } catch (BlobStorageException e) { AzureErrorType errorType = AzureUtils.parseErrorResponse(e); if (errorType != AzureErrorType.CONFLICT @@ -444,23 +446,15 @@ void pollStopSignals() { activeStreams.remove(idStr); continue; } - BlobClient stopBlob = containerClient.getBlobClient(locksPrefix + idStr + ".stop"); - if (Boolean.TRUE.equals(stopBlob.exists())) { + BlobClient stopBlob = getBlobClient(locksPrefix + idStr + ".stop"); + if (AzureUtils.blobExists(stopBlob)) { log.info("Detected remote .stop signal blob for upload ID {}", idStr); interruptLocalStream(idStr); - stopBlob.deleteIfExists(); + AzureUtils.deleteBlobQuietly(stopBlob); } } catch (Exception e) { log.debug("Error in azure-lock-watchdog polling execution: {}", e.getMessage()); } } } - - private String sanitizePrefix(String prefix) { - if (prefix == null || prefix.isEmpty()) { - return ""; - } - String result = prefix.startsWith("/") ? prefix.substring(1) : prefix; - return result.endsWith("/") ? result : result + "/"; - } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java index b5dc5de0..e3d0aae3 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobStorageService.java @@ -24,10 +24,12 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Base64; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.UUID; -import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -49,9 +51,9 @@ import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BoundedInputStream; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -148,10 +150,10 @@ public AzureBlobStorageService( Path tempBufferDir) { this.containerClient = Objects.requireNonNull(containerClient, "containerClient must not be null"); - this.uploadPrefix = sanitizePrefix(uploadPrefix); - this.metadataPrefix = sanitizePrefix(metadataPrefix); - this.checksumsPrefix = sanitizePrefix(checksumsPrefix); - this.locksPrefix = sanitizePrefix(locksPrefix); + this.uploadPrefix = AzureUtils.sanitizePrefix(uploadPrefix); + this.metadataPrefix = AzureUtils.sanitizePrefix(metadataPrefix); + this.checksumsPrefix = AzureUtils.sanitizePrefix(checksumsPrefix); + this.locksPrefix = AzureUtils.sanitizePrefix(locksPrefix); this.tempBufferDir = Objects.requireNonNull(tempBufferDir, "tempBufferDir must not be null"); try { @@ -162,19 +164,36 @@ public AzureBlobStorageService( Utils.cleanupTempFiles(this.tempBufferDir, "tus-azure-chunk-*.tmp", 24L * 3600_000L); AtomicInteger threadNum = new AtomicInteger(1); - this.uploadExecutor = + ThreadPoolExecutor pool = new ThreadPoolExecutor( cloudUploadThreadPoolSize, cloudUploadThreadPoolSize, 60L, TimeUnit.SECONDS, - new SynchronousQueue<>(), + new LinkedBlockingQueue<>(cloudUploadThreadPoolSize * 2), r -> { Thread t = new Thread(r, "tus-azure-upload-" + threadNum.getAndIncrement()); t.setDaemon(true); return t; }, - new ThreadPoolExecutor.CallerRunsPolicy()); + (r, executor) -> { + // CallerRunsPolicy must not be used: executing a multi-megabyte cloud upload + // synchronously on the servlet thread blocks the request thread from polling lock + // contention (waitForInFlight), preventing concurrent HEAD/DELETE interruptions. + // Instead, block on the queue until a worker thread becomes available or interrupt + // occurs. + try { + if (executor.isShutdown()) { + throw new RejectedExecutionException("Upload executor is shut down"); + } + executor.getQueue().put(r); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RejectedExecutionException("Interrupted while enqueuing upload task", e); + } + }); + pool.allowCoreThreadTimeOut(true); + this.uploadExecutor = pool; this.concatenationService = new AzureBlobConcatenationService(containerClient, this.uploadPrefix, this); @@ -204,12 +223,13 @@ public UploadInfo create(UploadInfo info, String ownerKey) throws IOException { info.setId(id); info.setOwnerKey(ownerKey); info.setOffset(0L); + info.setUploadPartKeys(new ArrayList<>()); // Set storageUploadId to the actual Azure Blob identifier (e.g. "uploads/") info.setStorageUploadId(getAzureBlobName(info)); if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { - info.setExpirationTimestamp(System.currentTimeMillis() + uploadExpirationPeriod); + info.updateExpiration(uploadExpirationPeriod); } // 2. Persist JSON metadata object to Azure (.info blob) @@ -231,6 +251,10 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) Objects.requireNonNull(upload, "UploadInfo must not be null"); Objects.requireNonNull(inputStream, "InputStream must not be null"); + if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { + upload.updateExpiration(uploadExpirationPeriod); + } + // 1. Obtain Azure Block Blob client and fetch pre-existing committed block list BlockBlobClient blockBlobClient = containerClient.getBlobClient(getAzureBlobName(upload)).getBlockBlobClient(); @@ -239,8 +263,10 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) // If metadata records committed uploadPartKeys, ensure the blob's committed blocks do not // exceed the authoritative metadata offset (e.g. if a prior commitBlockList succeeded // but updating UploadInfo metadata failed or was interrupted). - if (upload.getUploadPartKeys() != null && blockIds.size() > upload.getUploadPartKeys().size()) { - blockIds = new ArrayList<>(upload.getUploadPartKeys()); + List recordedPartKeys = + upload.getUploadPartKeys() != null ? upload.getUploadPartKeys() : Collections.emptyList(); + if (blockIds.size() > recordedPartKeys.size()) { + blockIds = new ArrayList<>(recordedPartKeys); blockBlobClient.commitBlockList(blockIds, true); } @@ -321,7 +347,7 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) break; } finally { if (!handedOff) { - deleteFileQuietly(chunkFile); + FileUtils.deleteQuietly(chunkFile); } } } @@ -363,11 +389,14 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) boolean finalComplete = upload.getLength() != null && upload.getOffset().equals(upload.getLength()); + boolean deduplicated = false; if (finalComplete) { - checkAndApplyDeduplication(upload); + deduplicated = checkAndApplyDeduplication(upload); } - saveUploadInfo(upload); + if (!deduplicated) { + saveUploadInfo(upload); + } } catch (Exception e) { log.error( "Failed to commit staged blocks or save metadata for upload ID {}", upload.getId(), e); @@ -487,7 +516,7 @@ public InputStream getUploadedBytes(UploadId id) throws IOException, UploadNotFo containerClient.getBlobClient(targetBlobName).getBlockBlobClient(); try { - if (Boolean.TRUE.equals(blockBlobClient.exists())) { + if (AzureUtils.blobExists(blockBlobClient)) { return blockBlobClient.openInputStream(); } } catch (Exception ignored) { @@ -554,7 +583,6 @@ public void terminateUpload(UploadInfo uploadInfo) throws UploadNotFoundExceptio deleteData(id); } deleteMetadata(id); - uploadLockingService.cleanupLock(id); log.debug("Terminated upload with ID {}", id); } @@ -575,7 +603,8 @@ public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) containerClient.getBlobClient(getAzureBlobName(uploadInfo)).getBlockBlobClient(); if (targetOffset == 0) { - blockBlobClient.deleteIfExists(); + AzureUtils.deleteBlobQuietly(blockBlobClient); + uploadInfo.setUploadPartKeys(new ArrayList<>()); } else { File tempFile = Files.createTempFile(tempBufferDir, "tus-azure-block-trim-", ".tmp").toFile(); try { @@ -592,8 +621,9 @@ public void removeLastNumberOfBytes(UploadInfo uploadInfo, long byteCount) blockBlobClient.stageBlock(newBlockId, is, tempFile.length()); } blockBlobClient.commitBlockList(List.of(newBlockId), true); + uploadInfo.setUploadPartKeys(new ArrayList<>(Collections.singletonList(newBlockId))); } finally { - deleteFileQuietly(tempFile); + FileUtils.deleteQuietly(tempFile); } } @@ -621,6 +651,10 @@ public void update(UploadInfo uploadInfo) throws IOException { Objects.requireNonNull(uploadInfo, "UploadInfo must not be null"); saveUploadInfo(uploadInfo); + if (uploadInfo.getDuplicatesUploadId() != null && deduplicationService != null) { + deduplicationService.syncParentExpirationIfChild(uploadInfo); + } + if (isUploadDeduplicationEnabled() && uploadInfo.getChecksum() != null && !uploadInfo.isUploadInProgress() @@ -634,19 +668,19 @@ public boolean dataExists(UploadId id) { if (id == null) { return false; } - return Boolean.TRUE.equals(containerClient.getBlobClient(uploadPrefix + id).exists()); + return AzureUtils.blobExists(containerClient, uploadPrefix + id); } @Override public void deleteData(UploadId id) { if (id != null) { - containerClient.getBlobClient(uploadPrefix + id).deleteIfExists(); + AzureUtils.deleteBlobQuietly(containerClient, uploadPrefix + id); } } private void deleteMetadata(UploadId id) { if (id != null) { - containerClient.getBlobClient(metadataPrefix + id + ".info").deleteIfExists(); + AzureUtils.deleteBlobQuietly(containerClient, metadataPrefix + id + ".info"); } } @@ -671,8 +705,12 @@ public void cleanupExpiredUploads() throws IOException { try { terminateUpload(info); } catch (UploadNotFoundException ignored) { - // Upload was already terminated or deleted concurrently + // Upload was already terminated or deleted concurrently; still ensure lock is cleaned } + this.uploadLockingService.cleanupLock(id); + } else if (info == null) { + // Corrupted or missing metadata object: clean up any orphaned lock + this.uploadLockingService.cleanupLock(id); } } } @@ -799,6 +837,10 @@ public int getCloudUploadThreadPoolSize() { return cloudUploadThreadPoolSize; } + ThreadPoolExecutor getUploadExecutor() { + return this.uploadExecutor; + } + @Override public void setDrainTimeout(Duration drainTimeout) { if (drainTimeout != null) { @@ -838,7 +880,7 @@ long calcOptimalBlockSize(Long totalLength) { /** Saves UploadInfo object as JSON in .info metadata blob. */ private void saveUploadInfo(UploadInfo info) throws IOException { String infoAsString = UploadInfoJsonSerializer.serialize(info); - if (StringUtils.isEmpty(infoAsString)) { + if (infoAsString == null || infoAsString.isEmpty()) { throw new IOException("Failed to serialize UploadInfo to JSON"); } byte[] jsonBytes = infoAsString.getBytes(StandardCharsets.UTF_8); @@ -973,32 +1015,25 @@ String generateBlockId(int index) { return Base64.getEncoder().encodeToString(idString.getBytes(StandardCharsets.UTF_8)); } - private void checkAndApplyDeduplication(UploadInfo uploadInfo) throws IOException { + private boolean checkAndApplyDeduplication(UploadInfo uploadInfo) throws IOException { if (!isUploadDeduplicationEnabled() || uploadInfo == null || uploadInfo.getChecksum() == null || uploadInfo.getChecksumAlgorithm() == null) { - return; + return false; } try { ChecksumKey key = ChecksumKey.of(uploadInfo.getChecksum(), uploadInfo.getChecksumAlgorithm()); deduplicate(uploadInfo, key); + return true; } catch (Exception e) { log.warn( "Failed to apply deduplication for upload {}: {}", uploadInfo.getId(), e.getMessage()); + return false; } } - /** Sanitizes object key prefix format. */ - private String sanitizePrefix(String prefix) { - if (prefix == null || prefix.isEmpty()) { - return ""; - } - String result = prefix.startsWith("/") ? prefix.substring(1) : prefix; - return result.endsWith("/") ? result : result + "/"; - } - /** Deletes local file suppressing exceptions. */ void deleteFileQuietly(File file) { if (file != null && file.exists()) { diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java index e9898f58..24ab642c 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureBlobUploadLock.java @@ -1,13 +1,17 @@ package me.desair.tus.server.upload.azure; +import com.azure.core.util.Context; import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.DeleteSnapshotsOptionType; import com.azure.storage.blob.specialized.BlobLeaseClient; import java.io.IOException; import java.lang.ref.WeakReference; import java.util.Map; import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -32,11 +36,12 @@ public class AzureBlobUploadLock implements UploadLock { private final BlobLeaseClient leaseClient; private final BlobClient lockBlob; + private final BlobClient stopBlob; private final String uploadUri; private final String uploadId; private final Map> activeStreams; private final Map activeLocks; - private final ScheduledExecutorService renewalExecutor; + private final ScheduledFuture renewalTask; private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); private boolean released = false; private boolean blobDeleted = false; @@ -137,30 +142,51 @@ public AzureBlobUploadLock( String uploadId, Map> activeStreams, Map activeLocks) { + this( + leaseClient, + lockBlob, + null, + uploadUri, + renewalExecutor, + uploadId, + activeStreams, + activeLocks); + } + + AzureBlobUploadLock( + BlobLeaseClient leaseClient, + BlobClient lockBlob, + BlobClient stopBlob, + String uploadUri, + ScheduledExecutorService renewalExecutor, + String uploadId, + Map> activeStreams, + Map activeLocks) { this.leaseClient = leaseClient; this.lockBlob = lockBlob; + this.stopBlob = stopBlob; this.uploadUri = Objects.requireNonNull(uploadUri, "uploadUri must not be null"); this.uploadId = uploadId; this.activeStreams = activeStreams; this.activeLocks = activeLocks; if (renewalExecutor != null) { - this.renewalExecutor = renewalExecutor; - } else if (leaseClient != null) { - // Initialize background daemon thread to renew lease periodically during upload - this.renewalExecutor = - Utils.scheduleWatchdog( - "azure-lease-renewal-" + uploadUri, + this.renewalTask = + renewalExecutor.scheduleWithFixedDelay( this::renewLease, RENEWAL_INTERVAL_SECONDS, RENEWAL_INTERVAL_SECONDS, TimeUnit.SECONDS); } else { - this.renewalExecutor = null; + this.renewalTask = null; } } - private static final long LEASE_DURATION_MS = 60000L; + // Azure Blob leases have a 30-second duration on the server. We track local expiration as + // 28 seconds (30s minus a 2s clock-drift safety margin) so that if connectivity to Azure is lost + // or renewal requests fail, we treat the lease as expired before Azure expires the lease on the + // server and grants it to another contender, preventing split-brain concurrent writes. + private static final long LEASE_DURATION_MS = 28000L; private long leaseExpiresAt = System.currentTimeMillis() + LEASE_DURATION_MS; BlobLeaseClient getLeaseClient() { @@ -216,10 +242,13 @@ void renewLease() { } catch (Exception e) { boolean leaseLost = false; if (e instanceof BlobStorageException) { - AzureErrorType errorType = AzureUtils.parseErrorResponse((BlobStorageException) e); + BlobStorageException bse = (BlobStorageException) e; + AzureErrorType errorType = AzureUtils.parseErrorResponse(bse); if (errorType == AzureErrorType.LEASE_ALREADY_PRESENT || errorType == AzureErrorType.LEASE_NOT_PRESENT - || errorType == AzureErrorType.CONFLICT) { + || errorType == AzureErrorType.CONFLICT + || bse.getStatusCode() == 409 + || bse.getStatusCode() == 412) { leaseLost = true; } } @@ -243,7 +272,7 @@ void renewLease() { } finally { thisObjectLock.writeLock().unlock(); } - shutdownExecutor(); + cancelRenewalTask(); // If lease renewal fails definitively, abort any active input stream immediately // so the upload thread does not continue writing un-locked data to Azure. if (activeStreams != null && uploadId != null) { @@ -282,7 +311,7 @@ public void release() { } finally { thisObjectLock.writeLock().unlock(); } - shutdownExecutor(); + cancelRenewalTask(); // Remove active stream and lock registrations from service maps upon release // to avoid stale references lingering in heap. if (activeStreams != null && uploadId != null) { @@ -298,17 +327,53 @@ public void release() { } finally { thisObjectLock.readLock().unlock(); } - if (leaseClient != null && !alreadyDeleted) { + + // Atomically delete the lock blob using the active lease ID. This guarantees that: + // 1. The lock blob is cleaned up immediately with a single HTTP round-trip (no leftover locks). + // 2. We are fenced against stale takeovers via Azure server-side condition (412 + // LeaseIdMismatch). + // 3. If deletion fails for any reason, we fall back to releasing the lease cleanly. + if (leaseClient != null && !alreadyDeleted && lockBlob != null) { try { - leaseClient.releaseLease(); - log.trace("Released Azure blob lease for upload URI {}", uploadUri); + lockBlob.deleteWithResponse( + DeleteSnapshotsOptionType.INCLUDE, + new BlobRequestConditions().setLeaseId(leaseClient.getLeaseId()), + null, + Context.NONE); + log.trace("Deleted Azure lock blob for upload URI {}", uploadUri); + } catch (BlobStorageException e) { + if (e.getStatusCode() != 404) { + log.debug( + "Azure lock blob delete failed (falling back to releaseLease) for URI {}: {}", + uploadUri, + e.getMessage()); + fallbackReleaseLease(); + } } catch (Exception e) { log.debug( - "Azure blob lease release failed (may have already expired/broken) for URI {}: {}", + "Azure lock blob delete failed (falling back to releaseLease) for URI {}: {}", uploadUri, e.getMessage()); + fallbackReleaseLease(); } } + + AzureUtils.deleteBlobQuietly(stopBlob); + } + + void fallbackReleaseLease() { + try { + executeReleaseLease(); + } catch (Exception ex) { + log.debug("Fallback releaseLease failed for URI {}: {}", uploadUri, ex.getMessage()); + } + } + + /** Performs the actual lease release network call against Azure SDK. */ + void executeReleaseLease() { + if (leaseClient != null) { + leaseClient.releaseLease(); + } } @Override @@ -321,8 +386,10 @@ public String getUploadUri() { return uploadUri; } - /** Shuts down the renewal executor cleanly. */ - private void shutdownExecutor() { - Utils.shutdownExecutor(renewalExecutor); + /** Cancels the periodic lease renewal task. */ + private void cancelRenewalTask() { + if (renewalTask != null) { + renewalTask.cancel(true); + } } } diff --git a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java index af579f28..84e71901 100644 --- a/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java +++ b/src/main/java/me/desair/tus/server/upload/azure/AzureUtils.java @@ -1,14 +1,143 @@ package me.desair.tus.server.upload.azure; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlockBlobClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** Utility helper methods for parsing and evaluating Azure Blob Storage error responses. */ +/** + * Utility helper methods for interacting with Azure Blob Storage and evaluating error responses. + */ public final class AzureUtils { + private static final Logger log = LoggerFactory.getLogger(AzureUtils.class); + private AzureUtils() { // Utility class } + /** + * Sanitizes a blob name prefix, stripping leading slashes, collapsing trailing slashes, and + * ensuring exactly one trailing slash. + * + * @param prefix Raw prefix string + * @return Normalized prefix ending with a slash, or empty string if null, empty, or only slashes + */ + public static String sanitizePrefix(String prefix) { + if (prefix == null || prefix.trim().isEmpty()) { + return ""; + } + String result = prefix.trim(); + while (result.startsWith("/")) { + result = result.substring(1); + } + if (result.isEmpty()) { + return ""; + } + while (result.endsWith("/")) { + result = result.substring(0, result.length() - 1); + } + return result + "/"; + } + + /** + * Checks whether a blob exists in Azure Blob Storage. + * + * @param blobClient The {@link BlobClient} to inspect + * @return {@code true} if the blob exists; {@code false} if it does not exist, argument is null, + * or an error occurs + */ + public static boolean blobExists(BlobClient blobClient) { + if (blobClient == null) { + return false; + } + try { + return Boolean.TRUE.equals(blobClient.exists()); + } catch (Exception e) { + return false; + } + } + + /** + * Checks whether a block blob exists in Azure Blob Storage. + * + * @param blobClient The {@link BlockBlobClient} to inspect + * @return {@code true} if the blob exists; {@code false} if it does not exist, argument is null, + * or an error occurs + */ + public static boolean blobExists(BlockBlobClient blobClient) { + if (blobClient == null) { + return false; + } + try { + return Boolean.TRUE.equals(blobClient.exists()); + } catch (Exception e) { + return false; + } + } + + /** + * Checks whether a blob with the given name exists in the specified container. + * + * @param containerClient The {@link BlobContainerClient} + * @param blobName The blob name to check + * @return {@code true} if the blob exists; {@code false} if it does not exist, any argument is + * null, or an error occurs + */ + public static boolean blobExists(BlobContainerClient containerClient, String blobName) { + if (containerClient == null || blobName == null) { + return false; + } + return blobExists(containerClient.getBlobClient(blobName)); + } + + /** + * Quietly deletes a blob if it exists without throwing exceptions, logging errors at debug level. + * + * @param blobClient The {@link BlobClient} to remove + */ + public static void deleteBlobQuietly(BlobClient blobClient) { + if (blobClient == null) { + return; + } + try { + blobClient.deleteIfExists(); + } catch (Exception e) { + log.debug("Failed to delete Azure blob quietly: {}", e.getMessage()); + } + } + + /** + * Quietly deletes a block blob if it exists without throwing exceptions. + * + * @param blobClient The {@link BlockBlobClient} to remove + */ + public static void deleteBlobQuietly(BlockBlobClient blobClient) { + if (blobClient == null) { + return; + } + try { + blobClient.deleteIfExists(); + } catch (Exception e) { + log.debug("Failed to delete Azure block blob quietly: {}", e.getMessage()); + } + } + + /** + * Quietly deletes a blob with the given name from the container without throwing exceptions. + * + * @param containerClient The {@link BlobContainerClient} + * @param blobName The blob name to remove + */ + public static void deleteBlobQuietly(BlobContainerClient containerClient, String blobName) { + if (containerClient == null || blobName == null) { + return; + } + deleteBlobQuietly(containerClient.getBlobClient(blobName)); + } + /** * Parses a {@link BlobStorageException} into a strongly-typed {@link AzureErrorType}. * diff --git a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java index a428212f..94830da0 100644 --- a/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingService.java @@ -329,11 +329,7 @@ public void requestLockRelease(String requestUri) { @Override public void cleanupLock(UploadId id) throws IOException { lockingServiceDelegate.cleanupLock(id); - } - - @Override - public void cleanupLock(String uploadUri) throws IOException { - lockingServiceDelegate.cleanupLock(uploadUri); + cleanupCache(); } @Override diff --git a/src/main/java/me/desair/tus/server/upload/checksum/DeduplicationService.java b/src/main/java/me/desair/tus/server/upload/checksum/DeduplicationService.java index 4fb28fd5..77661fae 100644 --- a/src/main/java/me/desair/tus/server/upload/checksum/DeduplicationService.java +++ b/src/main/java/me/desair/tus/server/upload/checksum/DeduplicationService.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import me.desair.tus.server.checksum.ChecksumKey; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; @@ -28,6 +29,7 @@ public class DeduplicationService { private final ChecksumIndex checksumIndex; private final UploadStorageService storageService; + private final Set inFlightDeduplications = ConcurrentHashMap.newKeySet(); private UploadLockingService lockingService; /** @@ -80,22 +82,32 @@ public UploadInfo deduplicate(UploadInfo uploadInfo, ChecksumKey key) throws IOE } UploadId currentId = uploadInfo.getId(); + if (currentId == null || !inFlightDeduplications.add(currentId)) { + // Re-entrant call detected: updateUploadInfo() calls storageService.update(), which invokes + // deduplicate() again. Guarding with inFlightDeduplications prevents infinite recursion + // (StackOverflowError) when duplicatesUploadId is still null. + return uploadInfo; + } - // 1. Check if a parent is currently registered for this checksum - UploadId parentId = checksumIndex.findParent(key); + try { + // 1. Check if a parent is currently registered for this checksum + UploadId parentId = checksumIndex.findParent(key); - if (parentId == null) { - // 2. No parent registered. Attempt CAS registration of current upload as parent. - return registerAsParent(uploadInfo, key); - } + if (parentId == null) { + // 2. No parent registered. Attempt CAS registration of current upload as parent. + return registerAsParent(uploadInfo, key); + } - // Guard: Do not deduplicate an upload against itself - if (Objects.equals(parentId, currentId)) { - return uploadInfo; - } + // Guard: Do not deduplicate an upload against itself + if (Objects.equals(parentId, currentId)) { + return uploadInfo; + } - // 3. Parent found ($P$). Acquire lock on $P$ to inspect and join. - return joinParentUnderLock(uploadInfo, key, parentId, true); + // 3. Parent found ($P$). Acquire lock on $P$ to inspect and join. + return joinParentUnderLock(uploadInfo, key, parentId, true); + } finally { + inFlightDeduplications.remove(currentId); + } } /** Registers the given upload as the parent for the specified checksum key. */ @@ -172,6 +184,69 @@ public UploadInfo getUploadInfoByChecksum(ChecksumKey key) throws IOException { return parentInfo; } + /** + * Acquires the upload lock on the specified parent upload with short bounded retries. + * + *

    When a parent completes, registerAsParent publishes the checksum index entry while the + * parent's request lock is still held by TusFileUploadService for a few milliseconds (running + * final post-handlers). A concurrent child or subsequent child completing milliseconds later + * would fail immediately without retry. Retrying up to 10 times with a 50ms pause bridges this + * transient completion window without deadlocking. + */ + private UploadLock lockParentWithRetry(UploadId parentId) throws IOException, TusException { + int retries = 0; + while (true) { + try { + return lockingService.lockUpload(parentId); + } catch (TusException e) { + if (++retries >= 10) { + throw e; + } + try { + Thread.sleep(50L); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw e; + } + } + } + } + + /** + * Synchronizes child upload expiration to its parent upload under parent lock. + * + *

    Ensures that the parent upload's expiration timestamp is at least as long as any of its + * active deduplicated children, preventing the parent data file from being prematurely expired + * and purged while active children still depend on it. + * + * @param childInfo the deduplicated child upload info + */ + public void syncParentExpirationIfChild(UploadInfo childInfo) { + if (childInfo == null || childInfo.getDuplicatesUploadId() == null) { + return; + } + UploadId parentId = childInfo.getDuplicatesUploadId(); + if (!inFlightDeduplications.add(parentId)) { + return; + } + try { + try (UploadLock parentLock = lockParentWithRetry(parentId)) { + UploadInfo parentInfo = storageService.getUploadInfo(parentId); + if (parentInfo != null) { + syncParentExpiration(parentInfo, childInfo); + } + } catch (Exception e) { + log.warn( + "Failed to sync expiration from child {} to parent {}: {}", + childInfo.getId(), + parentId, + e.getMessage()); + } + } finally { + inFlightDeduplications.remove(parentId); + } + } + /** * Acquires the lock on parent $P$, re-verifies parent state and physical bytes, and links the * child. @@ -179,7 +254,7 @@ public UploadInfo getUploadInfoByChecksum(ChecksumKey key) throws IOException { private UploadInfo joinParentUnderLock( UploadInfo uploadInfo, ChecksumKey key, UploadId expectedParentId, boolean allowRetry) throws IOException { - try (UploadLock parentLock = lockingService.lockUpload(expectedParentId)) { + try (UploadLock parentLock = lockParentWithRetry(expectedParentId)) { return doJoinParentUnderLock(uploadInfo, key, expectedParentId, allowRetry); } catch (TusException e) { // Parent is currently locked by active request. Per design, skip deduplication @@ -323,15 +398,47 @@ public void pruneOrphanedChecksumIndices() { for (ChecksumKey key : allKeys) { try { UploadId parentId = checksumIndex.findParent(key); - if (parentId != null) { - boolean parentDataExists = storageService.dataExists(parentId); - UploadInfo parentInfo = storageService.getUploadInfo(parentId); - boolean isTombstoned = checksumIndex.isParentDeleted(key); - boolean hasChildren = checksumIndex.hasChildren(key); + if (parentId == null) { + continue; + } + + // Quick unlocked pre-check + boolean parentDataExists = storageService.dataExists(parentId); + UploadInfo parentInfo = storageService.getUploadInfo(parentId); + boolean isTombstoned = checksumIndex.isParentDeleted(key); + boolean hasChildren = checksumIndex.hasChildren(key); + + boolean needsInspection = + !parentDataExists + || (parentInfo == null && !isTombstoned) + || (isTombstoned && !hasChildren); + + if (!needsInspection) { + continue; + } + + // Acquire lock on parent to prevent racing with concurrent children joining parent + try (UploadLock lock = lockingService.lockUpload(parentId)) { + // Re-verify under parent lock + UploadId currentParent = checksumIndex.findParent(key); + if (!Objects.equals(parentId, currentParent)) { + continue; + } + parentDataExists = storageService.dataExists(parentId); + parentInfo = storageService.getUploadInfo(parentId); + isTombstoned = checksumIndex.isParentDeleted(key); + hasChildren = checksumIndex.hasChildren(key); + + // Repair missing tombstone if parent info is gone but data exists and has children + if (parentDataExists && parentInfo == null && !isTombstoned && hasChildren) { + log.info("Repairing missing tombstone for parent {} with active children", parentId); + checksumIndex.markParentDeleted(key); + continue; + } boolean orphaned = !parentDataExists - || (parentInfo == null && !isTombstoned) + || (parentInfo == null && !hasChildren) || (isTombstoned && !hasChildren); if (orphaned) { @@ -344,6 +451,9 @@ public void pruneOrphanedChecksumIndices() { storageService.deleteData(parentId); } } + } catch (TusException e) { + // Parent is currently locked; skip this key for this sweep + log.debug("Parent {} is locked during prune check; skipping", parentId); } } catch (Exception e) { log.warn("Error checking checksum index for key {}: {}", key, e.getMessage()); @@ -403,26 +513,12 @@ private boolean removeChildFromIndex(UploadId childId, UploadId parentId, Checks * active children. */ private void purgeParentIfUnreferenced(ChecksumKey key, UploadId parentId) throws IOException { - try { - try (UploadLock parentLock = lockingService.lockUpload(parentId)) { - purgeParentUnderLock(key, parentId); - } catch (TusException e) { - log.warn( - "Could not acquire parent lock {} during final child cleanup; skipping immediate purge", - parentId); - } - } finally { - cleanupLock(parentId); - } - } - - private void cleanupLock(UploadId id) { - if (id != null) { - try { - lockingService.cleanupLock(id); - } catch (Exception e) { - log.warn("Failed to clean up lock for upload {}: {}", id, e.getMessage()); - } + try (UploadLock parentLock = lockParentWithRetry(parentId)) { + purgeParentUnderLock(key, parentId); + } catch (TusException e) { + log.warn( + "Could not acquire parent lock {} during final child cleanup; skipping immediate purge", + parentId); } } diff --git a/src/main/java/me/desair/tus/server/upload/disk/AbstractDiskBasedService.java b/src/main/java/me/desair/tus/server/upload/disk/AbstractDiskBasedService.java index 64c34e51..2ffe4cfe 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/AbstractDiskBasedService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/AbstractDiskBasedService.java @@ -3,19 +3,13 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import me.desair.tus.server.TusFileUploadService; import me.desair.tus.server.upload.AbstractCloseableResourceService; import me.desair.tus.server.upload.UploadId; -import me.desair.tus.server.util.Utils; import org.apache.commons.lang3.Validate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** Common abstract super class to implement service that use the disk file system */ public class AbstractDiskBasedService extends AbstractCloseableResourceService { - private static final Logger log = LoggerFactory.getLogger(TusFileUploadService.class); - private Path storagePath; public AbstractDiskBasedService(String path) { @@ -41,24 +35,12 @@ public Path getStoragePath() { protected Path getPathInStorageDirectory(UploadId id) { if (id == null) { return null; - } else { - Path path = storagePath.resolve(id.toString()); - if (!path.normalize().toAbsolutePath().startsWith(storagePath.normalize().toAbsolutePath())) { - throw new IllegalArgumentException( - "Upload ID is not valid and would result in a path traversal"); - } - return path; } + return DiskUtils.resolveSafePath( + storagePath, id.toString(), "Upload ID is not valid and would result in a path traversal"); } private synchronized void init() { - try { - Utils.ensureDirectoryExists(storagePath); - } catch (IOException e) { - String message = - "Unable to create the directory specified by the storage path " + storagePath; - log.error(message, e); - throw new StoragePathNotAvailableException(message, e); - } + DiskUtils.initStorageDirectory(storagePath); } } diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskChecksumIndex.java b/src/main/java/me/desair/tus/server/upload/disk/DiskChecksumIndex.java index 2dba4002..a371a5f7 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskChecksumIndex.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskChecksumIndex.java @@ -17,7 +17,6 @@ import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.checksum.ChecksumIndex; import me.desair.tus.server.util.Utils; -import org.apache.commons.io.FileUtils; /** * Disk-based implementation of {@link ChecksumIndex}. @@ -152,7 +151,7 @@ public void deleteIndex(ChecksumKey key) throws IOException { Objects.requireNonNull(key, "key must not be null"); Path checksumDir = getChecksumDir(key); if (Files.exists(checksumDir)) { - FileUtils.deleteDirectory(checksumDir.toFile()); + DiskUtils.deleteDirectoryQuietly(checksumDir); } } @@ -194,21 +193,17 @@ public Set getAllIndexedKeys() throws IOException { } private Path getChecksumDir(ChecksumKey key) { - Path path = storagePath.resolve(key.toString()); - if (!path.normalize().toAbsolutePath().startsWith(storagePath.normalize().toAbsolutePath())) { - throw new IllegalArgumentException( - "Checksum key is not valid and would result in a path traversal"); - } - return path; + return DiskUtils.resolveSafePath( + storagePath, + key.toString(), + "Checksum key is not valid and would result in a path traversal"); } private Path getChildMarker(Path childrenDir, UploadId childId) { Objects.requireNonNull(childId, "childId must not be null"); - Path path = childrenDir.resolve(childId.toString()); - if (!path.normalize().toAbsolutePath().startsWith(childrenDir.normalize().toAbsolutePath())) { - throw new IllegalArgumentException( - "Child upload ID is not valid and would result in a path traversal"); - } - return path; + return DiskUtils.resolveSafePath( + childrenDir, + childId.toString(), + "Child upload ID is not valid and would result in a path traversal"); } } diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java index 72433830..a2ef925f 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskLockingService.java @@ -7,7 +7,6 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.FileTime; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -36,11 +35,11 @@ public class DiskLockingService extends AbstractDiskBasedService implements Uplo private static final String LOCK_SUB_DIRECTORY = "locks"; /** Registry tracking active request input streams in the current JVM. */ - private static final ConcurrentHashMap> - activeLocks = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> activeLocks = + new ConcurrentHashMap<>(); - private static Thread watchdogThread = null; - private static final Object watchdogLock = new Object(); + private Thread watchdogThread = null; + private final Object watchdogLock = new Object(); private UploadIdFactory idFactory; @@ -58,7 +57,7 @@ public DiskLockingService(UploadIdFactory idFactory, String storagePath) { @Override protected void cleanupOnClose() throws IOException { synchronized (watchdogLock) { - Utils.interruptThread(watchdogThread); + DiskUtils.interruptThread(watchdogThread); watchdogThread = null; } for (WeakReference streamRef : activeLocks.values()) { @@ -106,18 +105,15 @@ private UploadLock lockUploadInternal(UploadId id, String uriOrId) public void cleanupStaleLocks() throws IOException { try (DirectoryStream locksStream = Files.newDirectoryStream(getStoragePath())) { for (Path path : locksStream) { - if (Files.exists(path)) { - FileTime lastModifiedTime = Files.getLastModifiedTime(path); - if (lastModifiedTime.toMillis() < System.currentTimeMillis() - 10000L) { - String fileName = path.getFileName().toString(); - if (fileName.endsWith(".stop")) { + if (Files.exists(path) && DiskUtils.isPathOlderThan(path, 10000L)) { + String fileName = path.getFileName().toString(); + if (fileName.endsWith(".stop")) { + Files.deleteIfExists(path); + } else { + UploadId id = new UploadId(fileName); + if (!isLocked(id)) { Files.deleteIfExists(path); - } else { - UploadId id = new UploadId(fileName); - if (!isLocked(id)) { - Files.deleteIfExists(path); - Files.deleteIfExists(path.resolveSibling(fileName + ".stop")); - } + Files.deleteIfExists(path.resolveSibling(fileName + ".stop")); } } } @@ -125,32 +121,6 @@ public void cleanupStaleLocks() throws IOException { } } - @Override - public void cleanupLock(UploadId id) throws IOException { - if (id == null) { - return; - } - // Only delete lock file if it exists and is not actively locked by another process/request. - // Notice we check Files.exists first because calling isLocked on a non-existent lock path - // would open a FileChannel with CREATE and instantiate a lock file on disk unnecessarily. - Path lockPath = getLockPath(id); - if (lockPath != null && Files.exists(lockPath)) { - if (!isLocked(id)) { - Files.deleteIfExists(lockPath); - } - } - Path stopPath = getStopPath(id); - if (stopPath != null) { - Files.deleteIfExists(stopPath); - } - } - - @Override - public void cleanupLock(String uploadUri) throws IOException { - UploadId id = idFactory != null ? idFactory.readUploadId(uploadUri) : null; - cleanupLock(id); - } - /** Checks whether the upload is locked by attempting to obtain a short-lived file lock. */ @Override public boolean isLocked(UploadId id) { @@ -158,6 +128,9 @@ public boolean isLocked(UploadId id) { Path lockPath = getLockPath(id); if (lockPath != null) { + if (!Files.exists(lockPath)) { + return false; + } // Try to obtain a lock to see if the upload is currently locked try (UploadLock lock = new FileBasedLock(id.toString(), lockPath)) { diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java index 25be91e7..68f95e7c 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskStorageService.java @@ -34,8 +34,6 @@ import me.desair.tus.server.upload.concatenation.VirtualConcatenationService; import me.desair.tus.server.util.UploadInfoJsonSerializer; import me.desair.tus.server.util.Utils; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -190,8 +188,9 @@ public UploadInfo getUploadInfoByChecksum(String checksum, ChecksumAlgorithm alg if (checksum == null || algorithm == null) { return null; } - String normalized = normalizeChecksum(checksum); - if (!isSafePathComponent(normalized) || !isSafePathComponent(algorithm.toString())) { + String normalized = DiskUtils.normalizeChecksum(checksum); + if (!DiskUtils.isSafePathComponent(normalized) + || !DiskUtils.isSafePathComponent(algorithm.toString())) { throw new IOException("The checksum contains an unsafe value."); } try { @@ -247,7 +246,7 @@ private void saveUploadInfo(UploadInfo info, Path path) throws IOException { if (isJsonSerializationEnabled()) { UploadInfoJsonSerializer.serializeToPath(info, path); } else { - Utils.writeSerializable(info, path); + DiskUtils.writeSerializable(info, path); } } @@ -263,9 +262,9 @@ private UploadInfo loadUploadInfo(Path path) throws IOException { e.getMessage()); info = null; } - return info != null ? info : Utils.readSerializable(path, UploadInfo.class); + return info != null ? info : DiskUtils.readSerializable(path, UploadInfo.class); } else { - return Utils.readSerializable(path, UploadInfo.class); + return DiskUtils.readSerializable(path, UploadInfo.class); } } @@ -291,6 +290,10 @@ public UploadInfo create(UploadInfo info, String ownerKey) throws IOException { info.setOffset(0L); info.setOwnerKey(ownerKey); + if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { + info.updateExpiration(uploadExpirationPeriod); + } + update(info); return info; @@ -316,44 +319,17 @@ public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundExce // It doesn't exist yet, which is fine } - // Retrieve the parent UploadInfo - UploadId parentId = uploadInfo.getDuplicatesUploadId(); - UploadInfo parentInfo = getUploadInfo(parentId); - if (parentInfo != null) { - Path parentBytesPath = null; - try { - parentBytesPath = getPathInUploadDir(parentId, DATA_FILE); - } catch (UploadNotFoundException e) { - // Parent file is not found - } - - if (parentBytesPath != null && Files.exists(parentBytesPath)) { - // Ensure parent's expiration timestamp is >= child's expiration timestamp - Long childExpire = uploadInfo.getExpirationTimestamp(); - Long parentExpire = parentInfo.getExpirationTimestamp(); - if (childExpire == null) { - if (parentExpire != null) { - parentInfo.setExpirationTimestamp(null); - Path parentInfoPath = getInfoPath(parentId); - saveUploadInfo(parentInfo, parentInfoPath); - } - } else { - if (parentExpire != null && parentExpire < childExpire) { - parentInfo.setExpirationTimestamp(childExpire); - Path parentInfoPath = getInfoPath(parentId); - saveUploadInfo(parentInfo, parentInfoPath); - } - } - } + if (deduplicationService != null) { + deduplicationService.syncParentExpirationIfChild(uploadInfo); } } else if (isUploadDeduplicationEnabled() && !uploadInfo.isUploadInProgress() && uploadInfo.getChecksum() != null && uploadInfo.getChecksumAlgorithm() != null && uploadInfo.getDuplicatesUploadId() == null) { - String normalized = normalizeChecksum(uploadInfo.getChecksum()); - if (!isSafePathComponent(normalized) - || !isSafePathComponent(uploadInfo.getChecksumAlgorithm().toString())) { + String normalized = DiskUtils.normalizeChecksum(uploadInfo.getChecksum()); + if (!DiskUtils.isSafePathComponent(normalized) + || !DiskUtils.isSafePathComponent(uploadInfo.getChecksumAlgorithm().toString())) { throw new IOException("The checksum contains an unsafe value."); } try { @@ -370,6 +346,10 @@ public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundExce public UploadInfo append(UploadInfo info, InputStream inputStream) throws IOException, TusException { if (info != null) { + if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { + info.updateExpiration(uploadExpirationPeriod); + } + Path bytesPath = getBytesPath(info.getId()); long max = getMaxUploadSize() > 0 ? getMaxUploadSize() : Long.MAX_VALUE; @@ -381,8 +361,9 @@ public UploadInfo append(UploadInfo info, InputStream inputStream) FileChannel file = FileChannel.open(bytesPath, WRITE)) { try { - // Lock will be released when the channel closes - file.lock(); + // Note: OS FileChannel.lock() is intentionally omitted here to prevent ENOLCK / + // "No locks available" errors on NFS mounts and Kubernetes RWX persistent volumes. + // Single-writer mutual exclusion is already guaranteed by the request-level UploadLock. // Validate that the given offset is at the end of the file if (!offset.equals(file.size())) { @@ -430,8 +411,7 @@ public void removeLastNumberOfBytes(UploadInfo info, long byteCount) try (FileChannel file = FileChannel.open(bytesPath, WRITE)) { - // Lock will be released when the channel closes - file.lock(); + // Note: OS FileChannel.lock() is intentionally omitted to avoid ENOLCK on NFS/K8s RWX PVCs. // Guard against Math.max(0, file.size() - byteCount) to prevent FileChannel.truncate // from throwing IllegalArgumentException on over-truncation during checksum rollback @@ -500,9 +480,9 @@ private void cleanDirectoryIfEmpty(UploadId id) { public void terminateUpload(UploadInfo info) throws UploadNotFoundException, IOException { if (info != null) { if (info.getChecksum() != null && info.getChecksumAlgorithm() != null) { - String normalized = normalizeChecksum(info.getChecksum()); - if (!isSafePathComponent(normalized) - || !isSafePathComponent(info.getChecksumAlgorithm().toString())) { + String normalized = DiskUtils.normalizeChecksum(info.getChecksum()); + if (!DiskUtils.isSafePathComponent(normalized) + || !DiskUtils.isSafePathComponent(info.getChecksumAlgorithm().toString())) { throw new IOException("The checksum contains an unsafe value."); } } @@ -519,10 +499,8 @@ public void terminateUpload(UploadInfo info) throws UploadNotFoundException, IOE Path uploadPath = getPathInStorageDirectory(info.getId()); if (uploadPath != null && deleteData) { - FileUtils.deleteDirectory(uploadPath.toFile()); + DiskUtils.deleteDirectoryQuietly(uploadPath); } - - uploadLockingService.cleanupLock(info.getId()); } } @@ -650,13 +628,12 @@ public void cleanupExpiredUploads() throws IOException { try { terminateUpload(info); } catch (UploadNotFoundException e) { - FileUtils.deleteDirectory(path.toFile()); - this.uploadLockingService.cleanupLock(id); + DiskUtils.deleteDirectoryQuietly(path); } } else { - FileUtils.deleteDirectory(path.toFile()); - this.uploadLockingService.cleanupLock(id); + DiskUtils.deleteDirectoryQuietly(path); } + uploadLockingService.cleanupLock(id); } } @@ -694,62 +671,6 @@ private Path getInfoPath(UploadId id) throws UploadNotFoundException { return getPathInUploadDir(id, INFO_FILE); } - private String normalizeChecksum(String checksum) { - String filename = checksum; - if (filename != null) { - if (!filename.matches("^[0-9a-fA-F]+$")) { - byte[] bytes = Base64.decodeBase64(filename); - if (bytes != null && bytes.length > 0) { - StringBuilder sb = new StringBuilder(); - for (byte b : bytes) { - sb.append(String.format("%02x", b)); - } - filename = sb.toString(); - } - } - } - return filename; - } - - /** - * Checks whether a given path component string is safe for use as a file or directory name on - * disk. - * - *

    OS differences in path component restrictions: - * - *

      - *
    • POSIX / Linux / macOS: Strict path separators are {@code /} and NUL ({@code \0}). - * Characters such as {@code ?}, {@code *}, {@code :}, {@code <}, {@code >}, {@code "}, and - * {@code |} are allowed in Linux filenames. - *
    • Windows (NTFS / FAT32): Reserved characters prohibited in file/directory names are - * {@code \}, {@code /}, {@code :}, {@code *}, {@code ?}, {@code "}, {@code <}, {@code >}, - * {@code |}, and ASCII control characters (0-31). Attempting to resolve paths containing - * these characters on Windows triggers an OS-level {@link - * java.nio.file.InvalidPathException}. - *
    - * - * To guarantee portable, cross-platform safety and prevent OS-dependent runtime exceptions on - * Windows, this method enforces validation against reserved characters from both Linux and - * Windows specifications, as well as relative path traversal sequences ({@code ..}). - * - * @param component the path component to validate - * @return {@code true} if the component is safe for cross-platform file system operations; {@code - * false} otherwise - */ - private boolean isSafePathComponent(String component) { - if (component == null || component.trim().isEmpty() || component.contains("..")) { - return false; - } - for (int i = 0; i < component.length(); i++) { - char c = component.charAt(i); - if (c < 32 || c == '/' || c == '\\' || c == ':' || c == '*' || c == '?' || c == '"' - || c == '<' || c == '>' || c == '|') { - return false; - } - } - return true; - } - private Path createUploadDirectory(UploadId id) throws IOException { Path dir = getPathInStorageDirectory(id); Utils.ensureDirectoryExists(dir); @@ -760,7 +681,7 @@ private Path getPathInUploadDir(UploadId id, String fileName) throws UploadNotFo // Get the upload directory Path uploadDir = getPathInStorageDirectory(id); if (uploadDir != null && Files.exists(uploadDir)) { - if (!isSafePathComponent(fileName)) { + if (!DiskUtils.isSafePathComponent(fileName)) { throw new IllegalArgumentException("The file name contains an unsafe value."); } return uploadDir.resolve(fileName); diff --git a/src/main/java/me/desair/tus/server/upload/disk/DiskUtils.java b/src/main/java/me/desair/tus/server/upload/disk/DiskUtils.java new file mode 100644 index 00000000..ebf46fd5 --- /dev/null +++ b/src/main/java/me/desair/tus/server/upload/disk/DiskUtils.java @@ -0,0 +1,378 @@ +package me.desair.tus.server.upload.disk; + +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectOutput; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.Serializable; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.util.Objects; +import java.util.UUID; +import me.desair.tus.server.util.Utils; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.serialization.ValidatingObjectInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Utility helper methods for disk-based storage, file locking, and path validation. */ +public final class DiskUtils { + + private static final Logger log = LoggerFactory.getLogger(DiskUtils.class); + private static final int LOCK_FILE_RETRY_COUNT = 3; + private static final long LOCK_FILE_SLEEP_TIME = 500; + + private DiskUtils() { + // Utility class + } + + /** + * Safely resolves a child path within a base directory, validating against directory traversal + * attempts. + * + * @param basePath The base parent directory path + * @param child The child filename or relative path + * @param errorMessage The error message to include if path traversal is detected + * @return The resolved child path, or null if basePath or child is null + * @throws IllegalArgumentException If path traversal is detected + */ + public static Path resolveSafePath(Path basePath, String child, String errorMessage) { + if (basePath == null || child == null) { + return null; + } + Path path = basePath.resolve(child); + if (!path.normalize().toAbsolutePath().startsWith(basePath.normalize().toAbsolutePath())) { + throw new IllegalArgumentException(errorMessage); + } + return path; + } + + /** + * Initializes a disk storage directory, ensuring it exists or throwing {@link + * StoragePathNotAvailableException} if directory creation fails. + * + * @param storagePath The directory path to initialize + * @throws StoragePathNotAvailableException If directory creation fails + */ + public static void initStorageDirectory(Path storagePath) { + Objects.requireNonNull(storagePath, "storagePath must not be null"); + try { + Utils.ensureDirectoryExists(storagePath); + } catch (IOException e) { + String message = + "Unable to create the directory specified by the storage path " + storagePath; + log.error(message, e); + throw new StoragePathNotAvailableException(message, e); + } + } + + /** + * Checks whether a given path component string is safe for use as a file or directory name on + * disk across platforms (preventing path traversal sequences and invalid characters). + * + * @param component The path component to validate + * @return {@code true} if safe; {@code false} otherwise + */ + public static boolean isSafePathComponent(String component) { + if (component == null || component.trim().isEmpty() || component.contains("..")) { + return false; + } + for (int i = 0; i < component.length(); i++) { + char c = component.charAt(i); + if (c < 32 || c == '/' || c == '\\' || c == ':' || c == '*' || c == '?' || c == '"' + || c == '<' || c == '>' || c == '|') { + return false; + } + } + return true; + } + + /** + * Normalizes a checksum value string, converting Base64-encoded checksums to lowercase hex string + * format if necessary. + * + * @param checksum Raw checksum string + * @return Normalized hex string checksum, or the original string if null or already hex + */ + public static String normalizeChecksum(String checksum) { + String filename = checksum; + if (filename != null) { + if (!filename.matches("^[0-9a-fA-F]+$")) { + byte[] bytes = Base64.decodeBase64(filename); + if (bytes != null && bytes.length > 0) { + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + filename = sb.toString(); + } + } + } + return filename; + } + + /** + * Checks whether a file or directory is older than a specified age in milliseconds relative to a + * given reference timestamp. + * + * @param path The file or directory path + * @param maxAgeMs The maximum age threshold in milliseconds + * @param now The reference timestamp in milliseconds + * @return {@code true} if older than or equal to maxAgeMs; {@code false} if younger, path is + * null, or an I/O exception occurs + */ + public static boolean isPathOlderThan(Path path, long maxAgeMs, long now) { + if (path == null) { + return false; + } + try { + FileTime mtime = Files.getLastModifiedTime(path); + return (now - mtime.toMillis()) >= maxAgeMs; + } catch (IOException e) { + return false; + } + } + + /** + * Checks whether a file or directory is older than a specified age in milliseconds relative to + * current system time. + * + * @param path The file or directory path + * @param maxAgeMs The maximum age threshold in milliseconds + * @return {@code true} if older than or equal to maxAgeMs; {@code false} otherwise + */ + public static boolean isPathOlderThan(Path path, long maxAgeMs) { + return isPathOlderThan(path, maxAgeMs, System.currentTimeMillis()); + } + + /** + * Recursively deletes a directory and all of its contents quietly without throwing exceptions. + * + * @param directory The directory path to delete recursively + * @return {@code true} if deleted successfully; {@code false} otherwise + */ + public static boolean deleteDirectoryQuietly(Path directory) { + if (directory == null || !Files.exists(directory)) { + return false; + } + try { + Files.walkFileTree( + directory, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + Utils.deletePathQuietly(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + Utils.deletePathQuietly(dir); + return FileVisitResult.CONTINUE; + } + }); + return !Files.exists(directory); + } catch (IOException e) { + log.debug("Failed to delete directory quietly: {}", directory, e); + return false; + } + } + + /** + * Resolves a unique temporary sibling path for a target destination file path. + * + * @param targetPath The destination file path + * @return A temporary sibling path with a unique UUID suffix, or null if targetPath is null + */ + public static Path createTempSiblingPath(Path targetPath) { + if (targetPath == null) { + return null; + } + Path parent = targetPath.getParent(); + String tempFileName = targetPath.getFileName().toString() + ".tmp." + UUID.randomUUID(); + return parent != null ? parent.resolve(tempFileName) : Paths.get(tempFileName); + } + + /** + * Creates an {@link AutoCloseable} {@link TempPath} that generates a unique temporary sibling + * path and automatically deletes it upon closing if it still exists. + * + * @param targetPath The destination file path + * @return An AutoCloseable TempPath instance + */ + public static TempPath createTempSibling(Path targetPath) { + return new TempPath(targetPath); + } + + /** + * AutoCloseable temporary sibling path that automatically cleans up (deletes) the temporary file + * upon closing if it still exists. + */ + public static class TempPath implements AutoCloseable { + private final Path path; + + /** + * Constructs a TempPath resolving a unique sibling path for the given target path. + * + * @param targetPath The destination file path + */ + public TempPath(Path targetPath) { + this.path = createTempSiblingPath(targetPath); + } + + /** + * Returns the underlying temporary sibling {@link Path}. + * + * @return The temporary path, or null if targetPath was null + */ + public Path getPath() { + return path; + } + + @Override + public void close() { + Utils.deletePathQuietly(path); + } + } + + /** + * Atomically moves a source file to a destination path, replacing any existing destination file. + * On network file systems (such as NFS mounts or Kubernetes Persistent Volumes) where {@code + * ATOMIC_MOVE} is unsupported by the filesystem driver, this method falls back gracefully to + * standard {@code REPLACE_EXISTING} move to prevent unexpected I/O failures. + * + * @param source The source file path to move + * @param destination The target destination file path + * @throws IOException If moving the file fails + */ + public static void atomicMove(Path source, Path destination) throws IOException { + if (source != null && destination != null) { + try { + Files.move( + source, + destination, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + // On network filesystems (e.g. NFS shares or Kubernetes PVs), ATOMIC_MOVE fails with + // AtomicMoveNotSupportedException. Fall back to standard REPLACE_EXISTING move. + Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING); + } + } + } + + /** + * Reads a serializable object from disk. + * + * @param Target object type + * @param path The file path to read from + * @param clazz The target object class + * @return Deserialized object instance, or null if reading fails or file does not exist + * @throws IOException If file access fails + */ + public static T readSerializable(Path path, Class clazz) throws IOException { + T info = null; + if (path != null && Files.exists(path)) { + try (InputStream is = Files.newInputStream(path); + ValidatingObjectInputStream ois = new ValidatingObjectInputStream(is)) { + ois.accept("java.lang.*", "java.util.*", "me.desair.tus.server.*"); + info = clazz.cast(ois.readObject()); + } catch (ClassNotFoundException | java.io.EOFException | java.io.StreamCorruptedException e) { + // File may be corrupted due to unexpected server shutdown + log.warn("Unable to read serializable file {}: {}", path, e.getMessage()); + info = null; + } + } + return info; + } + + /** + * Writes a serializable object to a file on disk atomically via a temporary file rename. + * + * @param object The serializable object to write + * @param path The file path to write to + * @throws IOException If file access or move fails + */ + public static void writeSerializable(Serializable object, Path path) throws IOException { + if (path != null && object != null) { + try (TempPath tempPath = new TempPath(path)) { + try (OutputStream os = + Files.newOutputStream(tempPath.getPath(), WRITE, CREATE, TRUNCATE_EXISTING); + OutputStream buffer = new BufferedOutputStream(os); + ObjectOutput output = new ObjectOutputStream(buffer)) { + + output.writeObject(object); + } + atomicMove(tempPath.getPath(), path); + } + } + } + + /** + * Locks the given file channel exclusively. + * + * @param channel The FileChannel to lock + * @return The acquired FileLock, or null if lock acquisition failed + * @throws IOException If an I/O error occurs + */ + public static FileLock lockFileExclusively(FileChannel channel) throws IOException { + return lockFile(channel, false); + } + + /** + * Locks the given file channel with a shared lock. + * + * @param channel The FileChannel to lock + * @return The acquired FileLock, or null if lock acquisition failed + * @throws IOException If an I/O error occurs + */ + public static FileLock lockFileShared(FileChannel channel) throws IOException { + return lockFile(channel, true); + } + + private static FileLock lockFile(FileChannel channel, boolean shared) throws IOException { + int i = 0; + FileLock lock = null; + do { + if (i > 0) { + Utils.sleep(LOCK_FILE_SLEEP_TIME); + } + + lock = channel.tryLock(0L, Long.MAX_VALUE, shared); + + i++; + } while (lock == null && i < LOCK_FILE_RETRY_COUNT); + + return lock; + } + + /** + * Safely interrupts and stops a thread, catching and logging any exceptions that occur. + * + * @param thread The Thread to interrupt + */ + public static void interruptThread(Thread thread) { + if (thread != null) { + try { + thread.interrupt(); + } catch (Exception e) { + log.debug("Error interrupting thread {}: {}", thread.getName(), e.getMessage()); + } + } + } +} diff --git a/src/main/java/me/desair/tus/server/upload/disk/FileBasedLock.java b/src/main/java/me/desair/tus/server/upload/disk/FileBasedLock.java index 3318f79a..6612f2fa 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/FileBasedLock.java +++ b/src/main/java/me/desair/tus/server/upload/disk/FileBasedLock.java @@ -48,7 +48,7 @@ private void tryToObtainFileLock() throws UploadAlreadyLockedException, IOExcept try { // Try to acquire a lock fileChannel = createFileChannel(); - FileLock fileLock = Utils.lockFileExclusively(fileChannel); + FileLock fileLock = DiskUtils.lockFileExclusively(fileChannel); // If the upload is already locked, our lock will be null if (fileLock == null) { @@ -92,9 +92,12 @@ public Path getLockPath() { @Override public void release() { try { - // Closing the channel will also release the lock - fileChannel.close(); + // On POSIX, delete the path before closing the channel so contenders do not lock an inode + // about to be unlinked. Utils.deletePathQuietly(lockPath); + if (fileChannel != null) { + fileChannel.close(); + } } catch (IOException e) { log.warn("Unable to release file lock for URI " + getUploadUri(), e); } diff --git a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java index 635ce45b..a91bf855 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java +++ b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileLockingService.java @@ -6,7 +6,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.attribute.FileTime; import me.desair.tus.server.upload.AbstractLeaseLockingService; import me.desair.tus.server.upload.LeaseData; import me.desair.tus.server.upload.UploadId; @@ -16,7 +15,6 @@ import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.util.LeaseDataJsonSerializer; import me.desair.tus.server.util.Utils; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,46 +136,18 @@ public void cleanupStaleLocks() throws IOException { atomicEvictExpiredLock(path); } } else if (fileName.endsWith(".mutex") && Files.isDirectory(path)) { - try { - FileTime mtime = Files.getLastModifiedTime(path); - if (now - mtime.toMillis() > 10_000L) { - FileUtils.deleteDirectory(path.toFile()); - } - } catch (IOException ignored) { - // Ignore transient cleanup error + if (DiskUtils.isPathOlderThan(path, 10_000L, now)) { + DiskUtils.deleteDirectoryQuietly(path); } } else if (fileName.endsWith(".stop") && Files.isRegularFile(path)) { - try { - FileTime mtime = Files.getLastModifiedTime(path); - if (now - mtime.toMillis() > 10_000L) { - Utils.deletePathQuietly(path); - } - } catch (IOException ignored) { - // Ignore transient cleanup error + if (DiskUtils.isPathOlderThan(path, 10_000L, now)) { + Utils.deletePathQuietly(path); } } } } } - @Override - public void cleanupLock(UploadId id) throws IOException { - if (id == null) { - return; - } - Path lockDirPath = getLockDirPath(id); - if (lockDirPath != null && Files.exists(lockDirPath)) { - // Safe eviction under sibling mutex to prevent deleting an active successor's lease - if (isLockExpired(id)) { - atomicEvictExpiredLock(lockDirPath); - } - } - Path stopFilePath = getStopFilePath(id); - if (stopFilePath != null) { - Utils.deletePathQuietly(stopFilePath); - } - } - /** * Attempts primary lock acquisition or in-place takeover of an expired lease under the protection * of a {@link LeaseFileMutex}. @@ -242,18 +212,13 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) thro } } catch (Exception e) { // Corrupted lease: check directory grace period - FileTime mtime = Files.getLastModifiedTime(lockDirPath); - if (now - mtime.toMillis() < EMPTY_DIR_GRACE_PERIOD_MS) { + if (!DiskUtils.isPathOlderThan(lockDirPath, EMPTY_DIR_GRACE_PERIOD_MS, now)) { return null; } } - } else { - // Empty directory: check directory grace period - FileTime mtime = Files.getLastModifiedTime(lockDirPath); - if (now - mtime.toMillis() < EMPTY_DIR_GRACE_PERIOD_MS) { - return null; - } } + // If lease.json does not exist (empty directory), no other node can be mid-creation + // because we hold the exclusive sibling LeaseFileMutex. Proceed with acquisition. } else { Utils.ensureDirectoryExists(lockDirPath); } @@ -270,7 +235,8 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) thro Utils.deletePathQuietly(stopFilePath); } - return new LeaseFileUploadLock(leaseData, lockDirPath, stopFilePath, activeInputStreams); + return new LeaseFileUploadLock( + leaseData, uploadId, lockDirPath, stopFilePath, activeInputStreams, watchdogExecutor); } catch (Exception e) { log.debug("Error acquiring lease for upload {}", uploadId, e); return null; @@ -309,8 +275,7 @@ protected void writeStopSignal(UploadId uploadId) { } @Override - protected void checkStopSignalForEntry(String uri, InputStream inputStream) { - UploadId uploadId = idFactory.readUploadId(uri); + protected void checkStopSignalForUpload(UploadId uploadId, InputStream inputStream) { if (uploadId == null) { return; } @@ -319,11 +284,21 @@ protected void checkStopSignalForEntry(String uri, InputStream inputStream) { if (stopFilePath != null && Files.exists(stopFilePath)) { log.info("Watchdog detected stop file for upload ID {}. Interrupting stream.", uploadId); Utils.interruptStream(inputStream); - activeInputStreams.remove(uri); + activeInputStreams.remove(uploadId); Utils.deletePathQuietly(stopFilePath); } } + @Override + protected void clearStopSignal(UploadId uploadId) { + if (uploadId != null) { + Path stopFilePath = getStopFilePath(uploadId); + if (stopFilePath != null) { + Utils.deletePathQuietly(stopFilePath); + } + } + } + /** * Determines if a lock directory is expired or abandoned. * @@ -354,16 +329,15 @@ boolean isLockDirectoryExpired(Path lockDirPath, long now) { } } - // If lease.json is missing or unparseable, check directory mtime against grace period - try { - FileTime mtime = Files.getLastModifiedTime(lockDirPath); - long dirAgeMs = now - mtime.toMillis(); - // If the directory was created within the last 5 seconds, another node is actively writing - return dirAgeMs >= EMPTY_DIR_GRACE_PERIOD_MS; - } catch (IOException e) { - // If directory is inaccessible or changing, assume active write in progress - return false; + // If lease.json is missing or unparseable, check if sibling mutex directory exists. + // If no sibling mutex is held, no node is mid-creation of lease.json, so the empty + // directory is unheld/abandoned (expired = true). + Path mutexDir = LeaseFileMutex.resolveMutexDir(lockDirPath); + if (mutexDir != null && Files.exists(mutexDir)) { + // Sibling mutex is actively held within grace window + return DiskUtils.isPathOlderThan(mutexDir, EMPTY_DIR_GRACE_PERIOD_MS, now); } + return true; } /** @@ -385,9 +359,9 @@ boolean atomicEvictExpiredLock(Path lockDirPath) { if (!isLockDirectoryExpired(lockDirPath, now)) { return false; } - FileUtils.deleteDirectory(lockDirPath.toFile()); + DiskUtils.deleteDirectoryQuietly(lockDirPath); return true; - } catch (IOException e) { + } catch (Exception e) { return false; } } @@ -396,34 +370,23 @@ Path getLockDirPath(final UploadId id) { if (id == null) { return null; } - Path path = storagePath.resolve(id.toString() + ".lock"); - if (!path.normalize().toAbsolutePath().startsWith(storagePath.normalize().toAbsolutePath())) { - throw new IllegalArgumentException( - "Upload ID is not valid and would result in a path traversal"); - } - return path; + return DiskUtils.resolveSafePath( + storagePath, + id.toString() + ".lock", + "Upload ID is not valid and would result in a path traversal"); } Path getStopFilePath(final UploadId id) { if (id == null) { return null; } - Path path = storagePath.resolve(id.toString() + ".stop"); - if (!path.normalize().toAbsolutePath().startsWith(storagePath.normalize().toAbsolutePath())) { - throw new IllegalArgumentException( - "Upload ID is not valid and would result in a path traversal"); - } - return path; + return DiskUtils.resolveSafePath( + storagePath, + id.toString() + ".stop", + "Upload ID is not valid and would result in a path traversal"); } private synchronized void initStoragePath() { - try { - Utils.ensureDirectoryExists(storagePath); - } catch (IOException e) { - String message = - "Unable to create the directory specified by the storage path " + storagePath; - log.error(message, e); - throw new StoragePathNotAvailableException(message, e); - } + DiskUtils.initStorageDirectory(storagePath); } } diff --git a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileMutex.java b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileMutex.java index e6618fbd..fa46c540 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileMutex.java +++ b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileMutex.java @@ -3,10 +3,8 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.FileTime; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.util.Utils; -import org.apache.commons.io.FileUtils; /** * An atomic filesystem mutex directory ({@code .mutex/}) providing mutual exclusion, @@ -47,12 +45,11 @@ public LeaseFileMutex(Path lockDirPath) { */ public LeaseFileMutex(final Path storagePath, final UploadId uploadId) { if (storagePath != null && uploadId != null) { - Path path = storagePath.resolve(uploadId.toString() + ".mutex"); - if (!path.normalize().toAbsolutePath().startsWith(storagePath.normalize().toAbsolutePath())) { - throw new IllegalArgumentException( - "Upload ID is not valid and would result in a path traversal"); - } - this.mutexDir = path; + this.mutexDir = + DiskUtils.resolveSafePath( + storagePath, + uploadId.toString() + ".mutex", + "Upload ID is not valid and would result in a path traversal"); } else { this.mutexDir = null; } @@ -109,11 +106,9 @@ private boolean tryAcquire() { return true; } catch (FileAlreadyExistsException e) { try { - FileTime mtime = Files.getLastModifiedTime(mutexDir); - long ageMs = System.currentTimeMillis() - mtime.toMillis(); - if (ageMs >= MUTEX_GRACE_PERIOD_MS) { + if (DiskUtils.isPathOlderThan(mutexDir, MUTEX_GRACE_PERIOD_MS)) { // Mutex holder crashed; clean up stale mutex directory and retry once - FileUtils.deleteDirectory(mutexDir.toFile()); + DiskUtils.deleteDirectoryQuietly(mutexDir); Files.createDirectory(mutexDir); return true; } diff --git a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileUploadLock.java b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileUploadLock.java index 815e4880..915afc77 100644 --- a/src/main/java/me/desair/tus/server/upload/disk/LeaseFileUploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/disk/LeaseFileUploadLock.java @@ -7,6 +7,7 @@ import java.util.concurrent.ScheduledExecutorService; import me.desair.tus.server.upload.AbstractLeaseLock; import me.desair.tus.server.upload.LeaseData; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.util.LeaseDataJsonSerializer; import me.desair.tus.server.util.Utils; import org.apache.commons.lang3.Strings; @@ -26,29 +27,31 @@ public class LeaseFileUploadLock extends AbstractLeaseLock { private final Path lockDirPath; private final Path stopFilePath; - private final LeaseData leaseData; /** * Constructs an active {@link LeaseFileUploadLock} and starts the background heartbeat lease * renewal daemon. * * @param leaseData The lease metadata + * @param uploadId The upload identifier associated with this lock * @param lockDirPath Dedicated lock directory path * @param stopFilePath Lock contention stop signal file path * @param activeInputStreams Map of active request input streams in the JVM */ public LeaseFileUploadLock( LeaseData leaseData, + UploadId uploadId, Path lockDirPath, Path stopFilePath, - Map activeInputStreams) { - this(leaseData, lockDirPath, stopFilePath, activeInputStreams, null); + Map activeInputStreams) { + this(leaseData, uploadId, lockDirPath, stopFilePath, activeInputStreams, null); } /** * Testing constructor allowing injection of a custom heartbeat executor. * * @param leaseData The lease metadata + * @param uploadId The upload identifier associated with this lock * @param lockDirPath Dedicated lock directory path * @param stopFilePath Lock contention stop signal file path * @param activeInputStreams Map of active request input streams in the JVM @@ -56,20 +59,16 @@ public LeaseFileUploadLock( */ LeaseFileUploadLock( LeaseData leaseData, + UploadId uploadId, Path lockDirPath, Path stopFilePath, - Map activeInputStreams, + Map activeInputStreams, ScheduledExecutorService heartbeatExecutor) { - super(leaseData, activeInputStreams, heartbeatExecutor, "lease-file-lock-heartbeat"); - this.leaseData = leaseData; + super(leaseData, uploadId, activeInputStreams, heartbeatExecutor); this.lockDirPath = lockDirPath; this.stopFilePath = stopFilePath; } - public LeaseData getLeaseData() { - return leaseData; - } - public String getStoragePath() { return lockDirPath != null ? lockDirPath.toString() : null; } @@ -89,17 +88,17 @@ protected void doRenewLease() throws Exception { log.warn("Failed to acquire mutex to renew lease for {}", lockDirPath); return; } + Path leaseFile = lockDirPath.resolve("lease.json"); // If another node took over the lease, abort renewal by throwing IllegalStateException // so AbstractLeaseLock terminates the heartbeat and aborts the active stream. - if (!doesLockOwnershipMatch()) { + if (Files.exists(leaseFile) && !doesLockOwnershipMatch()) { log.info("Lease for {} was taken over by another holder. Aborting renewal.", lockDirPath); throw new IllegalStateException( "Lease for " + lockDirPath + " was taken over by another holder"); } - Path leaseFile = lockDirPath.resolve("lease.json"); - leaseData.setExpiresAt(getExpiresAt()); - LeaseDataJsonSerializer.serializeToPath(leaseData, leaseFile); + getLeaseData().setExpiresAt(getExpiresAt()); + LeaseDataJsonSerializer.serializeToPath(getLeaseData(), leaseFile); } } @@ -112,9 +111,12 @@ protected void doRenewLease() throws Exception { protected void releaseLockResource() { if (lockDirPath != null && Files.exists(lockDirPath)) { try (LeaseFileMutex mutex = new LeaseFileMutex(lockDirPath)) { - if (mutex.isAcquired() && doesLockOwnershipMatch()) { - Utils.deletePathQuietly(lockDirPath.resolve("lease.json")); - Utils.deletePathQuietly(lockDirPath); + if (mutex.isAcquired()) { + Path leaseFile = lockDirPath.resolve("lease.json"); + if (!Files.exists(leaseFile) || doesLockOwnershipMatch()) { + Utils.deletePathQuietly(leaseFile); + Utils.deletePathQuietly(lockDirPath); + } } } } @@ -128,25 +130,27 @@ protected void releaseLockResource() { /** * Verifies that the on-disk {@code lease.json} file is still owned by this lock holder. * - * @return {@code true} if the lease file does not exist, is unparseable, or matches this holder's - * ID; {@code false} if owned by a different holder + *

    Fail-closed split-brain protection: returns {@code true} only when the lease file exists, is + * parseable, and its holder ID matches this lock holder. Returns {@code false} if the file is + * missing, unreadable, or owned by a different holder. + * + * @return {@code true} if the lease file exists and matches this holder's ID; {@code false} + * otherwise */ boolean doesLockOwnershipMatch() { if (lockDirPath == null) { return false; } Path leaseFile = lockDirPath.resolve("lease.json"); - if (Files.exists(leaseFile)) { - try { - LeaseData existingLease = LeaseDataJsonSerializer.deserialize(leaseFile); - if (existingLease != null - && !Strings.CS.equals(existingLease.getHolderId(), getHolderId())) { - return false; - } - } catch (Exception ignored) { - // If unparseable or corrupt, consider owned/proceed - } + if (!Files.exists(leaseFile)) { + return false; + } + try { + LeaseData existingLease = LeaseDataJsonSerializer.deserialize(leaseFile); + return existingLease != null && Strings.CS.equals(existingLease.getHolderId(), getHolderId()); + } catch (Exception e) { + log.warn("Error reading lease file {} for ownership verification", leaseFile, e); + return false; } - return true; } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ChecksumIndex.java b/src/main/java/me/desair/tus/server/upload/s3/S3ChecksumIndex.java index f47a2052..f23b95ec 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ChecksumIndex.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ChecksumIndex.java @@ -1,23 +1,15 @@ package me.desair.tus.server.upload.s3; -import io.minio.GetObjectArgs; import io.minio.ListObjectsArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import io.minio.RemoveObjectArgs; -import io.minio.RemoveObjectsArgs; import io.minio.Result; -import io.minio.StatObjectArgs; -import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; -import io.minio.messages.DeleteRequest; -import io.minio.messages.DeleteResult; import io.minio.messages.Item; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; @@ -25,6 +17,8 @@ import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.checksum.ChecksumKey; import me.desair.tus.server.upload.UploadId; @@ -81,7 +75,7 @@ public S3ChecksumIndex(MinioClient minioClient, String bucket) { public S3ChecksumIndex(MinioClient minioClient, String bucket, String checksumsPrefix) { this.minioClient = Objects.requireNonNull(minioClient, "MinioClient must not be null"); this.bucket = Objects.requireNonNull(bucket, "Bucket must not be null"); - this.checksumsPrefix = sanitizePrefix(checksumsPrefix); + this.checksumsPrefix = S3Utils.sanitizePrefix(checksumsPrefix); } /** @@ -174,8 +168,7 @@ public UploadId findParent(ChecksumKey key) throws IOException { Objects.requireNonNull(key, "key must not be null"); String parentKey = buildKey(key, PARENT_OBJECT); - try (InputStream stream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(parentKey).build())) { + try (InputStream stream = S3Utils.getObjectInputStream(minioClient, bucket, parentKey)) { String idStr = IOUtils.toString(stream, StandardCharsets.UTF_8).trim(); return idStr.isEmpty() ? null : new UploadId(idStr); } catch (ErrorResponseException e) { @@ -217,7 +210,7 @@ public void removeChild(ChecksumKey key, UploadId childId) throws IOException { Objects.requireNonNull(key, "key must not be null"); Objects.requireNonNull(childId, "childId must not be null"); String childKey = buildKey(key, CHILDREN_PREFIX + childId); - deleteObjectQuietly(childKey); + S3Utils.deleteS3ObjectQuietly(minioClient, bucket, childKey); } @Override @@ -230,7 +223,7 @@ public boolean hasChildren(ChecksumKey key) throws IOException { minioClient.listObjects( ListObjectsArgs.builder().bucket(bucket).prefix(prefix).maxKeys(1).build()); for (Result result : results) { - if (result.get() != null) { + if (result != null && result.get() != null) { return true; } } @@ -250,26 +243,13 @@ public void deleteIndex(ChecksumKey key) throws IOException { minioClient.listObjects( ListObjectsArgs.builder().bucket(bucket).prefix(prefix).recursive(true).build()); - List toDelete = new ArrayList<>(); - for (Result result : results) { - Item item = result.get(); - if (item != null && item.objectName() != null) { - toDelete.add(new DeleteRequest.Object(item.objectName())); - } - } + List toDelete = + StreamSupport.stream(results.spliterator(), false) + .map(S3Utils::extractObjectName) + .filter(Objects::nonNull) + .collect(Collectors.toList()); - if (!toDelete.isEmpty()) { - Iterable> errors = - minioClient.removeObjects( - RemoveObjectsArgs.builder().bucket(bucket).objects(toDelete).build()); - for (Result errorResult : errors) { - DeleteResult.Error error = errorResult.get(); - log.warn( - "Error deleting S3 object {} in checksum index: {}", - error.objectName(), - error.message()); - } - } + S3Utils.deleteS3ObjectsQuietly(minioClient, bucket, toDelete); } catch (Exception e) { throw new IOException("Failed to delete checksum index in S3 for prefix " + prefix, e); } @@ -321,15 +301,7 @@ private String buildKey(ChecksumKey key, String subPath) { } private boolean objectExists(String objectKey) { - try { - StatObjectResponse stat = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(objectKey).build()); - return stat != null; - } catch (ErrorResponseException e) { - return false; - } catch (Exception e) { - return false; - } + return S3Utils.objectExists(minioClient, bucket, objectKey); } private void putObjectUnconditional(String objectKey, byte[] data) throws IOException { @@ -342,25 +314,4 @@ private void putObjectUnconditional(String objectKey, byte[] data) throws IOExce throw new IOException("Failed to put object in S3: " + objectKey, e); } } - - private void deleteObjectQuietly(String objectKey) { - try { - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(objectKey).build()); - } catch (Exception ignored) { - } - } - - private static String sanitizePrefix(String prefix) { - if (prefix == null || prefix.trim().isEmpty()) { - return ""; - } - String trimmed = prefix.trim(); - while (trimmed.startsWith("/")) { - trimmed = trimmed.substring(1); - } - if (!trimmed.isEmpty() && !trimmed.endsWith("/")) { - trimmed = trimmed + "/"; - } - return trimmed; - } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java index b6207758..60f959b8 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3ConcatenationService.java @@ -145,7 +145,7 @@ public S3ConcatenationService( S3ServerSideComposeHelper s3ComposeHelper) { this.minioClient = Objects.requireNonNull(minioClient, "MinioClient must not be null"); this.bucket = Objects.requireNonNull(bucket, "Bucket must not be null"); - this.objectPrefix = objectPrefix != null ? objectPrefix : ""; + this.objectPrefix = S3Utils.sanitizePrefix(objectPrefix); this.uploadStorageService = Objects.requireNonNull(uploadStorageService, "UploadStorageService must not be null"); this.temporaryDirectory = @@ -159,12 +159,7 @@ public S3ConcatenationService( private static MinioClient buildMinioClient( String endpoint, String region, String accessKey, String secretKey) { - String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; - return MinioClient.builder() - .endpoint(endpoint) - .credentials(accessKey, secretKey) - .region(effectiveRegion) - .build(); + return S3Utils.buildMinioClient(endpoint, region, accessKey, secretKey); } @Override @@ -289,7 +284,19 @@ private void mergeUsingServerSideCopy( try { List partKeys = new ArrayList<>(); for (UploadInfo partial : partialUploads) { - partKeys.add(partial.getStorageUploadId()); + // Resolve duplicate child uploads dynamically to the parent upload's physical S3 object + // key to prevent composing against non-existent child blob targets. + String key; + if (uploadStorageService instanceof S3StorageService) { + key = ((S3StorageService) uploadStorageService).getS3ObjectKey(partial); + } else if (partial.getDuplicatesUploadId() != null) { + key = objectPrefix + partial.getDuplicatesUploadId(); + } else if (partial.getStorageUploadId() != null) { + key = partial.getStorageUploadId(); + } else { + key = objectPrefix + partial.getId(); + } + partKeys.add(key); } // Execute S3 server-side object composition via native S3 multipart copy helper. diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java index 48586424..7f374c73 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3LockingService.java @@ -1,17 +1,19 @@ package me.desair.tus.server.upload.s3; -import io.minio.GetObjectArgs; +import com.fasterxml.jackson.core.JsonProcessingException; import io.minio.ListObjectsArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import io.minio.RemoveObjectArgs; import io.minio.Result; -import io.minio.StatObjectArgs; import io.minio.errors.ErrorResponseException; import io.minio.messages.Item; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Objects; import java.util.concurrent.locks.ReadWriteLock; @@ -59,6 +61,7 @@ public class S3LockingService extends AbstractLeaseLockingService { public static final String DEFAULT_LOCKS_PREFIX = "locks/"; public static final long DEFAULT_LEASE_DURATION_MS = 30_000L; // 30 seconds public static final long DEFAULT_POLL_INTERVAL_MS = 2_000L; // 2 seconds + public static final long MUTEX_GRACE_PERIOD_MS = 5_000L; // 5 seconds for stale eviction mutexes private final MinioClient minioClient; private final String bucket; @@ -232,9 +235,22 @@ public void cleanupStaleLocks() throws IOException { for (Result result : results) { Item item = result.get(); - // Remove expired .lock lease objects - if (item.objectName().endsWith(".lock") && isLockExpired(item.objectName())) { - deleteObjectQuietly(item.objectName()); + if (item == null || item.objectName() == null) { + continue; + } + String name = item.objectName(); + // 1. Remove expired .lock lease objects + if (name.endsWith(".lock") && isLockExpired(name)) { + deleteObjectQuietly(name); + } + // 2. Remove stale .stop signal objects older than 30s + else if (name.endsWith(".stop")) { + ZonedDateTime lastModified = item.lastModified(); + if (lastModified != null + && lastModified.isBefore(ZonedDateTime.now().minusSeconds(30L))) { + deleteObjectQuietly(name); + log.info("Cleaned up stale S3 stop signal object: {}", name); + } } } } catch (Exception e) { @@ -243,22 +259,13 @@ public void cleanupStaleLocks() throws IOException { } @Override - public void cleanupLock(UploadId id) throws IOException { - if (id == null) { - return; - } - String lockKey = buildLockKey(id); - String stopKey = buildStopKey(id); - - // Only delete the lock object if it is expired or already absent - if (isLockExpired(lockKey)) { - deleteObjectQuietly(lockKey); - } - deleteObjectQuietly(stopKey); + protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { + return tryAcquireLock(uploadId, leaseData, false); } @Override - protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { + protected UploadLock tryAcquireLock( + UploadId uploadId, LeaseData leaseData, boolean isEvictionRetry) { if (leaseData == null) { return null; } @@ -287,11 +294,11 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { thisObjectLock.readLock().unlock(); } + boolean conditionalWriteUsed = false; if (conditionalSupported) { // Layer 1: Optimistic Conditional PutObject with "If-None-Match: *" // AWS S3 and compliant servers reject this with 412 Precondition Failed if the object - // already - // exists + // already exists try { minioClient.putObject( PutObjectArgs.builder() @@ -303,6 +310,7 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { (long) lockContentBytes.length, -1L) .build()); + conditionalWriteUsed = true; } catch (ErrorResponseException e) { S3ErrorType errorType = S3Utils.parseErrorResponse(e); if (errorType == S3ErrorType.API_NOT_IMPLEMENTED) { @@ -336,16 +344,29 @@ protected UploadLock tryAcquireLock(UploadId uploadId, LeaseData leaseData) { writeUnconditionalLockObject(lockKey, lockContentBytes); } - // Layer 2: Jittered Read-After-Write Verification - // For non-CAS backends or backends where If-None-Match is not strictly enforced, - // pause for a randomized jitter duration to allow competing writes to settle, - // then verify our holderId is still the owner - applyJitter(); - if (!verifyLockOwnership(lockKey, leaseData.getHolderId())) { - return null; + // If conditional write is supported and was used, S3 guarantees atomic CAS creation + // on the happy path where no eviction occurred (a competing write fails immediately + // with 412 PreconditionFailed). We can skip the sleep jitter and extra GET verification, + // saving 20-60ms latency and 1 network RTT. + // However, if an expired lock eviction occurred (isEvictionRetry == true) or on non-CAS + // backends (conditionalWriteUsed == false), multiple contenders may have raced during + // eviction deletion. We must apply jitter and verify ownership to prevent TOCTOU split-brain. + if (!conditionalWriteUsed || isEvictionRetry) { + applyJitter(); + if (!verifyLockOwnership(lockKey, leaseData.getHolderId())) { + return null; + } } - return new S3UploadLock(leaseData, minioClient, bucket, lockKey, stopKey, activeInputStreams); + return new S3UploadLock( + leaseData, + uploadId, + minioClient, + bucket, + lockKey, + stopKey, + activeInputStreams, + watchdogExecutor); } catch (ErrorResponseException e) { S3ErrorType errorType = S3Utils.parseErrorResponse(e); if (errorType == S3ErrorType.PRECONDITION_FAILED || errorType == S3ErrorType.CONFLICT) { @@ -382,27 +403,40 @@ protected boolean evictExpiredLock(UploadId uploadId) { return false; } String lockKey = buildLockKey(uploadId); - try (InputStream stream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(lockKey).build())) { - LeaseData lock = LeaseDataJsonSerializer.deserialize(stream); - if (!isLeaseExpired(lock, System.currentTimeMillis())) { - return false; - } - // Actual expired lock found in S3: delete it so next tryAcquireLock() can succeed - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(lockKey).build()); - log.info("Evicted expired S3 lock for key {}", lockKey); - return true; - } catch (ErrorResponseException e) { - if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { - // Lock key was already released normally and removed from S3. - // Return true to indicate the resource is clear without emitting false eviction logs. + String mutexKey = buildMutexKey(uploadId); + + // Acquire atomic S3 eviction mutex to serialize concurrent contenders across pods/threads. + // Without mutual exclusion, multiple contenders can inspect an expired lock simultaneously, + // and a lagging contender's unconditioned removeObject call would delete a newly acquired, + // active lock written by the winning contender (causing split-brain lock collisions). + if (!tryAcquireMutex(mutexKey)) { + return false; + } + + try { + // Under exclusive mutex, inspect the lease in S3 to ensure it is still expired + try (InputStream stream = S3Utils.getObjectInputStream(minioClient, bucket, lockKey)) { + LeaseData lock = LeaseDataJsonSerializer.deserialize(stream); + if (!isLeaseExpired(lock, System.currentTimeMillis())) { + return false; + } + // Actual expired lock found in S3: delete it so next tryAcquireLock() can succeed + S3Utils.deleteS3Object(minioClient, bucket, lockKey); + log.info("Evicted expired S3 lock for key {}", lockKey); return true; + } catch (ErrorResponseException e) { + if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { + // Lock key was already evicted or released normally; resource is clear + return true; + } + log.warn("Failed checking S3 lock expiration for key {}", lockKey, e); + return false; } - log.warn("Failed checking S3 lock expiration for key {}", lockKey, e); - return false; } catch (Exception e) { log.warn("Failed to evict expired S3 lock for key {}", lockKey, e); return false; + } finally { + deleteObjectQuietly(mutexKey); } } @@ -420,34 +454,31 @@ protected void writeStopSignal(UploadId uploadId) { } @Override - protected void checkStopSignalForEntry(String uri, InputStream inputStream) { - UploadId uploadId = idFactory.readUploadId(uri); + protected void checkStopSignalForUpload(UploadId uploadId, InputStream inputStream) { if (uploadId == null) { return; } String stopKey = buildStopKey(uploadId); - try { - // Check if a .stop signal object was written by another pod requesting lock release - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(stopKey).build()); + // Check if a .stop signal object was written by another pod requesting lock release + if (S3Utils.objectExists(minioClient, bucket, stopKey)) { // Remote stop signal object found! Interrupt local byte stream immediately Utils.interruptStream(inputStream); // Clean up the stream registration and delete the stop signal to avoid duplicate interrupts - activeInputStreams.remove(uri); + activeInputStreams.remove(uploadId); deleteObjectQuietly(stopKey); - } catch (ErrorResponseException e) { - if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { - // Normal state: no stop signal object in S3 - return; - } - } catch (Exception e) { - log.debug("Error checking stop signal for {}", stopKey, e); + } + } + + @Override + protected void clearStopSignal(UploadId uploadId) { + if (uploadId != null) { + deleteS3ObjectsQuietly(Arrays.asList(buildStopKey(uploadId), buildMutexKey(uploadId))); } } boolean verifyLockOwnership(String lockKey, String expectedHolderId) { - try (InputStream stream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(lockKey).build())) { + try (InputStream stream = S3Utils.getObjectInputStream(minioClient, bucket, lockKey)) { LeaseData remoteLock = LeaseDataJsonSerializer.deserialize(stream); return remoteLock != null && Strings.CS.equals(remoteLock.getHolderId(), expectedHolderId) @@ -460,10 +491,19 @@ boolean verifyLockOwnership(String lockKey, String expectedHolderId) { } boolean isLockExpired(String lockKey) { - try (InputStream stream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(lockKey).build())) { + try (InputStream stream = S3Utils.getObjectInputStream(minioClient, bucket, lockKey)) { + + LeaseData lock; + try { + lock = LeaseDataJsonSerializer.deserialize(stream); + } catch (JsonProcessingException | IllegalArgumentException e) { + log.warn("Corrupted JSON in lock object {}, treating as expired", lockKey, e); + return true; + } - LeaseData lock = LeaseDataJsonSerializer.deserialize(stream); + if (lock == null) { + return true; + } // Add a 2000ms safety buffer beyond expiration time to absorb NTP clock // drift between distributed pods before considering a lease expired. return isLeaseExpired(lock, System.currentTimeMillis()); @@ -471,19 +511,35 @@ boolean isLockExpired(String lockKey) { if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { return true; // Key missing -> Not locked } - return true; + // Fail-closed split-brain protection: on non-NoSuchKey S3 errors (e.g. 500 InternalError, + // 503 SlowDown), assume the lock is still actively held by another node. + log.warn( + "S3 error reading lock object {}, assuming lock is still active: {}", + lockKey, + e.getMessage()); + return false; } catch (Exception e) { - log.debug("Failed to read lock object {}, treating as expired", lockKey, e); - return true; + // Fail-closed split-brain protection: on transient network/IO exceptions, assume the lock + // is still actively held by another node. + log.warn("Failed to read lock object {}, assuming lock is still active", lockKey, e); + return false; } } + /** + * Quietly deletes multiple S3 objects in a single batch request using S3 Multi-Object Delete. + * + *

    If multi-object delete encounters an error (e.g. bucket policy restriction), falls back to + * individual deletions quietly. + * + * @param keys The S3 object keys to delete + */ + void deleteS3ObjectsQuietly(Collection keys) { + S3Utils.deleteS3ObjectsQuietly(minioClient, bucket, keys); + } + private void deleteObjectQuietly(String key) { - try { - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); - } catch (Exception e) { - log.debug("Failed to delete S3 object key {}", key, e); - } + S3Utils.deleteS3ObjectQuietly(minioClient, bucket, key); } @Override @@ -492,11 +548,7 @@ protected void applyJitter() { } private String sanitizePrefix(String prefix) { - if (prefix == null || prefix.isEmpty()) { - return ""; - } - String result = prefix.startsWith("/") ? prefix.substring(1) : prefix; - return result.endsWith("/") ? result : result + "/"; + return S3Utils.sanitizePrefix(prefix); } private String buildLockKey(UploadId uploadId) { @@ -507,13 +559,165 @@ private String buildStopKey(UploadId uploadId) { return locksPrefix + uploadId.toString() + ".stop"; } + private String buildMutexKey(UploadId uploadId) { + return locksPrefix + uploadId.toString() + ".mutex"; + } + + /** + * Attempts to acquire the exclusive S3 eviction mutex for an upload. + * + *

    Eviction mutex prevents TOCTOU race conditions where multiple pods concurrently inspect an + * expired lock and subsequent lagging contenders delete an active lock newly written by the + * winning contender. + * + * @param mutexKey S3 object key for the eviction mutex + * @return {@code true} if mutex acquired; {@code false} if actively held by another contender + */ + private boolean tryAcquireMutex(String mutexKey) { + if (mutexKey == null) { + return false; + } + + // Step 1: On S3 backends supporting atomic conditional writes, attempt CAS acquisition + if (isS3ConditionalWritesSupported()) { + return tryAcquireConditionalMutex(mutexKey); + } + + // Step 2: On non-CAS backends (e.g. Wasabi, Ceph), fall back to safe pre-checked write + return tryAcquireNonCasMutex(mutexKey); + } + + /** + * Attempts atomic eviction mutex acquisition using an S3 conditional write (If-None-Match: *). + * + *

    If the mutex object already exists (HTTP 412/409), checks whether the existing mutex was + * abandoned by a crashed contender and attempts stale recovery. + * + * @param mutexKey S3 object key for the eviction mutex + * @return {@code true} if acquired or recovered; {@code false} if held by a live contender + */ + private boolean tryAcquireConditionalMutex(String mutexKey) { + byte[] nowBytes = createMutexPayload(); + try { + writeConditionalMutexObject(mutexKey, nowBytes); + return true; + } catch (ErrorResponseException e) { + S3ErrorType errorType = S3Utils.parseErrorResponse(e); + + // S3 endpoint does not support conditional writes: downgrade and fall back to non-CAS + if (errorType == S3ErrorType.API_NOT_IMPLEMENTED) { + handleConditionalWriteNotImplemented(mutexKey); + return tryAcquireNonCasMutex(mutexKey); + } + + // Mutex collision: another pod holds the mutex. Inspect age for crash recovery + if (errorType == S3ErrorType.PRECONDITION_FAILED || errorType == S3ErrorType.CONFLICT) { + return recoverStaleConditionalMutex(mutexKey, nowBytes); + } + + // Unexpected error response (e.g. AccessDenied): fail closed to prevent split-brain + return false; + } catch (Exception e) { + // Network or I/O failure: fail closed to prevent split-brain + return false; + } + } + + /** + * Attempts to recover an eviction mutex abandoned by a crashed or hung contender. + * + *

    If the existing mutex object is older than {@link #MUTEX_GRACE_PERIOD_MS} (5 seconds), it is + * treated as abandoned, deleted, and re-acquired atomically. + * + * @param mutexKey S3 object key for the eviction mutex + * @param nowBytes Current timestamp payload + * @return {@code true} if stale mutex was recovered; {@code false} if actively held + */ + private boolean recoverStaleConditionalMutex(String mutexKey, byte[] nowBytes) { + if (!isMutexStale(mutexKey)) { + // Mutex was created within grace period: active contender is mid-eviction + return false; + } + + // Mutex is older than 5s grace period: previous holder crashed or hung; clear and retry + deleteObjectQuietly(mutexKey); + try { + writeConditionalMutexObject(mutexKey, nowBytes); + return true; + } catch (Exception retryEx) { + // Lost race against another recovering node + return false; + } + } + + /** + * Attempts eviction mutex acquisition on S3 backends lacking conditional writes. + * + * @param mutexKey S3 object key for the eviction mutex + * @return {@code true} if acquired; {@code false} if actively held or I/O fails + */ + private boolean tryAcquireNonCasMutex(String mutexKey) { + if (!isMutexStale(mutexKey)) { + // Existing mutex is active within grace period + return false; + } + try { + writeUnconditionalMutexObject(mutexKey, createMutexPayload()); + return true; + } catch (Exception e) { + log.debug("Failed writing non-CAS eviction mutex for key {}", mutexKey, e); + return false; + } + } + + private void writeConditionalMutexObject(String mutexKey, byte[] payload) throws Exception { + minioClient.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(mutexKey) + .extraHeaders(Collections.singletonMap("If-None-Match", "*")) + .stream(new ByteArrayInputStream(payload), (long) payload.length, -1L) + .build()); + } + + private void writeUnconditionalMutexObject(String mutexKey, byte[] payload) throws Exception { + minioClient.putObject( + PutObjectArgs.builder().bucket(bucket).object(mutexKey).stream( + new ByteArrayInputStream(payload), (long) payload.length, -1L) + .build()); + } + + private byte[] createMutexPayload() { + return String.valueOf(System.currentTimeMillis()).getBytes(StandardCharsets.UTF_8); + } + + private void handleConditionalWriteNotImplemented(String mutexKey) { + log.info( + "S3 endpoint does not support conditional writes on mutex. " + + "Downgrading to non-CAS arbitration for mutex {}", + mutexKey); + thisObjectLock.writeLock().lock(); + try { + s3ConditionalWritesSupported = false; + } finally { + thisObjectLock.writeLock().unlock(); + } + } + + private boolean isMutexStale(String mutexKey) { + try (InputStream stream = S3Utils.getObjectInputStream(minioClient, bucket, mutexKey)) { + byte[] bytes = stream.readAllBytes(); + long createdTime = Long.parseLong(new String(bytes, StandardCharsets.UTF_8).trim()); + return (System.currentTimeMillis() - createdTime) >= MUTEX_GRACE_PERIOD_MS; + } catch (ErrorResponseException e) { + return S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY; + } catch (Exception e) { + return true; + } + } + private static MinioClient buildMinioClient( String endpoint, String region, String accessKey, String secretKey) { - String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; - return MinioClient.builder() - .endpoint(endpoint) - .credentials(accessKey, secretKey) - .region(effectiveRegion) - .build(); + return S3Utils.buildMinioClient(endpoint, region, accessKey, secretKey); } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java index b062aca8..483d65e3 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3StorageService.java @@ -1,17 +1,10 @@ package me.desair.tus.server.upload.s3; -import io.minio.GetObjectArgs; import io.minio.ListObjectsArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import io.minio.RemoveObjectArgs; -import io.minio.RemoveObjectsArgs; import io.minio.Result; -import io.minio.StatObjectArgs; -import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; -import io.minio.messages.DeleteRequest; -import io.minio.messages.DeleteResult; import io.minio.messages.Item; import java.io.ByteArrayInputStream; import java.io.File; @@ -27,7 +20,6 @@ import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Enumeration; @@ -35,12 +27,15 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.UUID; -import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.checksum.ChecksumKey; import me.desair.tus.server.exception.MaxAppendSizeExceededException; @@ -64,6 +59,7 @@ import me.desair.tus.server.util.Utils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.BoundedInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -137,7 +133,6 @@ public class S3StorageService implements UploadStorageService { private UploadConcatenationService concatenationService; private final ReadWriteLock thisObjectLock = new ReentrantReadWriteLock(); private boolean s3ComposeObjectSupported = true; - private boolean supportsBatchDelete = true; private S3ServerSideComposeHelper s3ComposeHelper; /** @@ -231,10 +226,10 @@ public S3StorageService( S3ServerSideComposeHelper s3ComposeHelper) { this.minioClient = Objects.requireNonNull(minioClient, "MinioClient must not be null"); this.bucket = Objects.requireNonNull(bucket, "Bucket must not be null"); - this.objectPrefix = sanitizePrefix(objectPrefix); - this.metadataPrefix = sanitizePrefix(metadataPrefix); - this.checksumsPrefix = sanitizePrefix(checksumsPrefix); - this.locksPrefix = sanitizePrefix(locksPrefix); + this.objectPrefix = S3Utils.sanitizePrefix(objectPrefix); + this.metadataPrefix = S3Utils.sanitizePrefix(metadataPrefix); + this.checksumsPrefix = S3Utils.sanitizePrefix(checksumsPrefix); + this.locksPrefix = S3Utils.sanitizePrefix(locksPrefix); this.temporaryDirectory = temporaryDirectory != null ? temporaryDirectory @@ -249,19 +244,36 @@ public S3StorageService( } AtomicInteger threadNum = new AtomicInteger(1); - this.uploadExecutor = + ThreadPoolExecutor pool = new ThreadPoolExecutor( cloudUploadThreadPoolSize, cloudUploadThreadPoolSize, 60L, TimeUnit.SECONDS, - new SynchronousQueue<>(), + new LinkedBlockingQueue<>(cloudUploadThreadPoolSize * 2), r -> { Thread t = new Thread(r, "tus-s3-upload-" + threadNum.getAndIncrement()); t.setDaemon(true); return t; }, - new ThreadPoolExecutor.CallerRunsPolicy()); + (r, executor) -> { + // CallerRunsPolicy must not be used: executing a multi-megabyte cloud upload + // synchronously on the servlet thread blocks the request thread from polling lock + // contention (waitForInFlight), preventing concurrent HEAD/DELETE interruptions. + // Instead, block on the queue until a worker thread becomes available or interrupt + // occurs. + try { + if (executor.isShutdown()) { + throw new RejectedExecutionException("Upload executor is shut down"); + } + executor.getQueue().put(r); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RejectedExecutionException("Interrupted while enqueuing upload task", e); + } + }); + pool.allowCoreThreadTimeOut(true); + this.uploadExecutor = pool; this.s3ComposeHelper = s3ComposeHelper != null ? s3ComposeHelper : new S3ServerSideComposeHelper(this.minioClient); @@ -284,12 +296,7 @@ public S3StorageService( private static MinioClient buildMinioClient( String endpoint, String region, String accessKey, String secretKey) { - String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; - return MinioClient.builder() - .endpoint(endpoint) - .credentials(accessKey, secretKey) - .region(effectiveRegion) - .build(); + return S3Utils.buildMinioClient(endpoint, region, accessKey, secretKey); } /** @@ -304,7 +311,7 @@ public String getS3ObjectKey(UploadInfo uploadInfo) { return null; } - // Resolve duplicate child upload dynamically to parent S3 object key per AGENTS.md §7 + // Resolve duplicate child upload dynamically to parent S3 object key if (uploadInfo.getDuplicatesUploadId() != null) { return buildObjectKey(uploadInfo.getDuplicatesUploadId()); } @@ -369,8 +376,7 @@ public UploadInfo getUploadInfo(UploadId id) throws IOException { String metadataKey = buildMetadataKey(id); String json; // Step 1: Read JSON metadata object from S3 (/.info) - try (InputStream stream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(metadataKey).build())) { + try (InputStream stream = S3Utils.getObjectInputStream(minioClient, bucket, metadataKey)) { json = IOUtils.toString(stream, StandardCharsets.UTF_8); } catch (ErrorResponseException e) { // Return null if key does not exist in S3 @@ -417,6 +423,12 @@ public UploadInfo create(UploadInfo info, String ownerKey) throws IOException { info.setOffset(0L); } + // Pre-calculate expiration if configured so initial metadata write already includes it, + // avoiding a duplicate PUT metadata round-trip in ExpirationRequestHandler. + if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { + info.updateExpiration(uploadExpirationPeriod); + } + // Persist initial UploadInfo metadata object (.info) to S3 try { update(info); @@ -429,14 +441,28 @@ public UploadInfo create(UploadInfo info, String ownerKey) throws IOException { @Override public UploadInfo append(UploadInfo upload, InputStream inputStream) throws IOException, TusException { - // Step 1: Verify upload existence and check configured size limits - UploadInfo info = fetchAndValidateUpload(upload.getId()); - if (upload.getLength() != null && info.getLength() == null) { + // Step 1: Verify upload existence and check configured size limits. + // Reuse the caller-provided UploadInfo instance directly when available to avoid an + // expensive remote S3 GET (metadata/.info) on every PATCH chunk. + UploadInfo info = upload; + if (info == null || info.getId() == null || info.getStorageUploadId() == null) { + info = fetchAndValidateUpload(upload != null ? upload.getId() : null); + } else { + validateUploadLimits(info); + } + if (upload != null && upload.getLength() != null && info.getLength() == null) { info.setLength(upload.getLength()); } String objectKey = getS3ObjectKey(info); InterruptibleInputStream interruptibleStream = Utils.toInterruptibleStream(inputStream); + // Pre-calculate expiration if configured so updated metadata already reflects the renewed + // expiration, avoiding an extra remote PUT metadata round-trip in ExpirationRequestHandler. + if (uploadExpirationPeriod != null && uploadExpirationPeriod > 0) { + info.updateExpiration(uploadExpirationPeriod); + upload.updateExpiration(uploadExpirationPeriod); + } + // Step 2: If the last part is sub-5MB, download & prepend its bytes to the incoming stream. // The previous tail part is intentionally kept in S3 as a staleTailKey and is ONLY deleted // after the new manifest is successfully committed to S3. This ensures zero data loss if @@ -448,9 +474,13 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) validateRemainingPartBudget(info, preparedStream.remainingPartKeys.size()); // Clean up any unmanifested orphan part objects left in S3 from earlier aborted attempts. - // Pruning unmanifested parts under the upload lock prevents listing delays and ensures - // that only parts explicitly tracked in the manifest are considered valid. - pruneOrphanParts(info.getId(), preparedStream.remainingPartKeys, preparedStream.staleTailKeys); + // Only prune orphaned unmanifested parts if prior aborted tail parts need replacement. + // Skipping unmanifested part listing on clean, healthy sequential chunks eliminates an + // expensive S3 ListObjectsV2 network round-trip per chunk. + if (!preparedStream.staleTailKeys.isEmpty()) { + pruneOrphanParts( + info.getId(), preparedStream.remainingPartKeys, preparedStream.staleTailKeys); + } // Step 3: Process payload stream in optimal chunk parts and upload to S3. AppendResult appendResult = processPayloadChunks(info, preparedStream, info.getId()); @@ -473,7 +503,13 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) upload.setOffset(newOffset); info.setUploadPartKeys(new ArrayList<>(appendResult.allPartKeys)); upload.setUploadPartKeys(new ArrayList<>(appendResult.allPartKeys)); - update(info); + + // If multi-part parts need to be composed later, persist manifest once before compose + // so crash-recovery metadata is durable in S3. For single-chunk direct uploads (where + // allPartKeys is empty), defer metadata persistence until after the chunk is confirmed. + if (!appendResult.allPartKeys.isEmpty()) { + update(info); + } // Safe transactional cleanup: Delete old tail parts only AFTER the new manifest is saved. // If the upload was interrupted earlier, staleTailKeys were untouched, preserving bytes. @@ -510,8 +546,12 @@ public UploadInfo append(UploadInfo upload, InputStream inputStream) } // Step 6: If all expected bytes are uploaded, compose all part chunks into final S3 object - finalizeCompletedUploadIfFinished(info, objectKey, info.getId(), currentOffset); - update(info); + if (isCompletingOrEmpty) { + finalizeCompletedUploadIfFinished(info, objectKey, info.getId(), currentOffset); + update(info); + } else if (appendResult.allPartKeys.isEmpty() && hasNewParts) { + update(info); + } return info; } @@ -538,8 +578,30 @@ private void writeMetadataObject(UploadInfo uploadInfo) throws IOException { @Override public void update(UploadInfo uploadInfo) throws IOException, UploadNotFoundException { + if (uploadInfo == null) { + return; + } + + // Finalize deferred-length S3 uploads when completed via update(). + // When length is announced after append (e.g. Tus 1.0.0 Upload-Defer-Length: 1 or RUFH chunked + // requests with Upload-Complete: ?1), finalizeCompletedUploadIfFinished composes parts into the + // final destination object (uploads/) so dataExists() and downloads succeed. + if (!uploadInfo.isUploadInProgress() + && uploadInfo.getDuplicatesUploadId() == null + && uploadInfo.getUploadPartKeys() != null + && !uploadInfo.getUploadPartKeys().isEmpty()) { + String objectKey = getS3ObjectKey(uploadInfo); + long currentOffset = uploadInfo.getOffset() != null ? uploadInfo.getOffset() : 0L; + finalizeCompletedUploadIfFinished(uploadInfo, objectKey, uploadInfo.getId(), currentOffset); + } + writeMetadataObject(uploadInfo); + // Sync parent expiration if this is a deduplicated child upload + if (uploadInfo.getDuplicatesUploadId() != null && deduplicationService != null) { + deduplicationService.syncParentExpirationIfChild(uploadInfo); + } + // Index checksum for deduplication if upload is completed and deduplication is enabled if (isUploadDeduplicationEnabled() && uploadInfo.getChecksum() != null @@ -621,6 +683,10 @@ public void cleanupExpiredUploads() throws IOException { && info.isExpired() && !this.uploadLockingService.isLocked(id)) { terminateUpload(info); + this.uploadLockingService.cleanupLock(id); + } else if (info == null) { + // Corrupted or missing metadata object: clean up any orphaned lock + this.uploadLockingService.cleanupLock(id); } } } @@ -726,8 +792,6 @@ public void terminateUpload(UploadInfo uploadInfo) throws UploadNotFoundExceptio deleteData(uploadInfo.getId()); } deleteMetadata(uploadInfo.getId()); - - uploadLockingService.cleanupLock(uploadInfo.getId()); } @Override @@ -907,6 +971,10 @@ public int getCloudUploadThreadPoolSize() { return cloudUploadThreadPoolSize; } + ThreadPoolExecutor getUploadExecutor() { + return this.uploadExecutor; + } + @Override public void setDrainTimeout(Duration drainTimeout) { if (drainTimeout != null) { @@ -945,24 +1013,6 @@ public long getPreferredPartSize() { return preferredPartSize; } - boolean isSupportsBatchDelete() { - thisObjectLock.readLock().lock(); - try { - return supportsBatchDelete; - } finally { - thisObjectLock.readLock().unlock(); - } - } - - void setSupportsBatchDelete(boolean supportsBatchDelete) { - thisObjectLock.writeLock().lock(); - try { - this.supportsBatchDelete = supportsBatchDelete; - } finally { - thisObjectLock.writeLock().unlock(); - } - } - @Override public void close() throws IOException { uploadExecutor.shutdown(); @@ -1032,29 +1082,42 @@ private PreparedStream prepareStreamWithExistingIncompletePart( // 2. Roll back any existing sub-5MB numbered parts into the stream buffer so they can grow if (!existingPartKeys.isEmpty()) { String lastPartKey = existingPartKeys.get(existingPartKeys.size() - 1); - try { - StatObjectResponse stat = - minioClient.statObject( - StatObjectArgs.builder().bucket(bucket).object(lastPartKey).build()); - if (stat.size() < minPartSize) { - log.info( - "Rolling back sub-5MB part {} (size: {}) into stream buffer", - lastPartKey, - stat.size()); + long lastPartSize = getPartSize(lastPartKey); + if (lastPartSize > 0 && lastPartSize < minPartSize) { + log.info( + "Rolling back sub-5MB part {} (size: {}) into stream buffer", + lastPartKey, + lastPartSize); + // Stream into a secure temp file instead of buffering up to 5 MB directly in JVM heap + File tempPrepFile = + Files.createTempFile(temporaryDirectory, "tus-s3-prep-", ".tmp").toFile(); + try { try (InputStream lastPartStream = - minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(lastPartKey).build())) { - byte[] lastPartBytes = IOUtils.toByteArray(lastPartStream); - prependedBytes += lastPartBytes.length; - combinedStream = - new SequenceInputStream(new ByteArrayInputStream(lastPartBytes), combinedStream); - staleTailKeys.add(lastPartKey); - existingPartsTotalSize -= lastPartBytes.length; - existingPartKeys.remove(existingPartKeys.size() - 1); + S3Utils.getObjectInputStream(minioClient, bucket, lastPartKey); + FileOutputStream fos = new FileOutputStream(tempPrepFile)) { + IOUtils.copyLarge(lastPartStream, fos); } + long tempPrepLength = tempPrepFile.length(); + prependedBytes += tempPrepLength; + InputStream prependedStream = + new FileInputStream(tempPrepFile) { + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + FileUtils.deleteQuietly(tempPrepFile); + } + } + }; + combinedStream = new SequenceInputStream(prependedStream, combinedStream); + staleTailKeys.add(lastPartKey); + existingPartsTotalSize -= tempPrepLength; + existingPartKeys.remove(existingPartKeys.size() - 1); + } catch (Exception e) { + FileUtils.deleteQuietly(tempPrepFile); + log.debug("Error inspecting last part key {}: {}", lastPartKey, e.getMessage()); } - } catch (Exception e) { - log.debug("Error inspecting last part key {}: {}", lastPartKey, e.getMessage()); } } @@ -1067,19 +1130,36 @@ private PreparedStream prepareStreamWithExistingIncompletePart( staleTailKeys); } + /** + * Calculates total size across committed parts in O(1) time per part key by reading the size + * encoded in the part key name, avoiding O(N) sequential S3 statObject HTTP calls. + */ private long calculateTotalPartsSize(List partKeys) { long totalSize = 0L; for (String key : partKeys) { + totalSize += getPartSize(key); + } + return totalSize; + } + + /** + * Resolves the byte size of an S3 part key. If the key encodes its size suffix (_), it is + * extracted in O(1) time without any network calls; otherwise falls back to statObject. + */ + private long getPartSize(String partKey) { + if (partKey == null) { + return 0L; + } + int underscoreIdx = partKey.lastIndexOf('_'); + if (underscoreIdx >= 0 && underscoreIdx < partKey.length() - 1) { try { - StatObjectResponse stat = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(key).build()); - if (stat != null) { - totalSize += stat.size(); - } - } catch (Exception ignored) { + return Long.parseLong(partKey.substring(underscoreIdx + 1)); + } catch (NumberFormatException ignored) { + // Fall back to statObject if suffix is not numeric } } - return totalSize; + long size = S3Utils.getObjectSize(minioClient, bucket, partKey); + return size >= 0 ? size : 0L; } /** @@ -1089,8 +1169,7 @@ private long calculateTotalPartsSize(List partKeys) { * @param manifestedParts Parts actively tracked in the manifest * @param staleTailKeys Stale tail parts pending replacement in this request */ - private void pruneOrphanParts( - UploadId id, List manifestedParts, List staleTailKeys) { + void pruneOrphanParts(UploadId id, List manifestedParts, List staleTailKeys) { List s3Parts = fetchExistingPartKeys(id); for (String s3Part : s3Parts) { if (!manifestedParts.contains(s3Part) && !staleTailKeys.contains(s3Part)) { @@ -1185,7 +1264,7 @@ private AppendResult processPayloadChunks( plannedPartKeys.clear(); plannedChunkSizes.clear(); - String chunkKey = buildChunkPartKey(id, nextPartNumber++); + String chunkKey = buildChunkPartKey(id, nextPartNumber++, chunkBytesWritten); uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); allPartKeys.add(chunkKey); confirmedBytesAppended += chunkBytesWritten; @@ -1194,6 +1273,33 @@ private AppendResult processPayloadChunks( break; } + // Single-chunk direct upload fast-path: + // If this is the first chunk of the upload, the stream finished cleanly, and this chunk + // covers 100% of the declared upload length, stream directly into the final destination + // object key (uploads/). This bypasses multi-step S3 multipart upload and + // eliminates 5 sequential S3 HTTP calls (PUT .part, HEAD .part, CreateMultipartUpload, + // UploadPartCopy, CompleteMultipartUpload, DELETE .part). + boolean isFirstChunk = allPartKeys.isEmpty() && plannedPartKeys.isEmpty(); + boolean isCleanFinished = + streamFinished + && readException == null + && maxAppendSizeException == null + && !preparedStream.isInterrupted(); + boolean isFullSingleChunk = + isFirstChunk + && isCleanFinished + && info.hasLength() + && chunkBytesWritten == info.getLength(); + + if (isFullSingleChunk) { + String targetKey = getS3ObjectKey(info); + uploadChunkToS3(targetKey, tempChunkFile, chunkBytesWritten); + info.setStorageUploadId(targetKey); + confirmedBytesAppended += chunkBytesWritten; + handedOff = true; + break; + } + long currentTotalOffset = baseOffset + totalBytesAppended; boolean isUploadComplete = readException == null @@ -1206,7 +1312,7 @@ private AppendResult processPayloadChunks( if (chunkBytesWritten >= minPartSize) { // Full part chunk (>= 5 MB): Submit to AsyncChunkUploader pipeline for background // upload - String chunkKey = buildChunkPartKey(id, nextPartNumber++); + String chunkKey = buildChunkPartKey(id, nextPartNumber++, chunkBytesWritten); plannedPartKeys.add(chunkKey); plannedChunkSizes.add(chunkBytesWritten); File fileToUpload = tempChunkFile; @@ -1231,7 +1337,7 @@ private AppendResult processPayloadChunks( plannedPartKeys.clear(); plannedChunkSizes.clear(); - String chunkKey = buildChunkPartKey(id, nextPartNumber++); + String chunkKey = buildChunkPartKey(id, nextPartNumber++, chunkBytesWritten); uploadChunkToS3(chunkKey, tempChunkFile, chunkBytesWritten); allPartKeys.add(chunkKey); confirmedBytesAppended += chunkBytesWritten; @@ -1330,17 +1436,10 @@ private void finalizeCompletedUploadIfFinished( boolean canUseServerSideCompose = true; if (partKeys.size() > 1) { // AWS S3 requires all parts except the last to be >= 5 MB (5,242,880 bytes) for - // composeObject + // composeObject. We check sizes using getPartSize() in O(1) time without statObject. for (int i = 0; i < partKeys.size() - 1; i++) { - try { - StatObjectResponse stat = - minioClient.statObject( - StatObjectArgs.builder().bucket(bucket).object(partKeys.get(i)).build()); - if (stat.size() < minPartSize) { - canUseServerSideCompose = false; - break; - } - } catch (Exception e) { + long size = getPartSize(partKeys.get(i)); + if (size < minPartSize) { canUseServerSideCompose = false; break; } @@ -1393,21 +1492,6 @@ private void finalizeCompletedUploadIfFinished( throw new IOException("Failed to create empty completed object for ID " + id, e); } } - - // Add checksum index if deduplication is enabled - if (isUploadDeduplicationEnabled() - && info.getChecksum() != null - && info.getDuplicatesUploadId() == null) { - try { - ChecksumKey key = ChecksumKey.of(info.getChecksum(), info.getChecksumAlgorithm()); - deduplicate(info, key); - } catch (Exception e) { - log.warn( - "Failed to register checksum index for completed upload {}: {}", - info.getId(), - e.getMessage()); - } - } } } @@ -1436,8 +1520,7 @@ private InputStream fetchS3ByteStream(UploadId id, UploadInfo info) try { // Step 1: Attempt to read from completed object key in S3 - return minioClient.getObject( - GetObjectArgs.builder().bucket(bucket).object(objectKey).build()); + return S3Utils.getObjectInputStream(minioClient, bucket, objectKey); } catch (ErrorResponseException e) { if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { // Step 2: Fallback to reading from manifest part objects if upload is in-progress @@ -1484,9 +1567,7 @@ private void truncateManifestParts(UploadInfo uploadInfo, long bytesToRemove) { while (!partKeys.isEmpty() && bytesToRemoveRemaining > 0) { String lastKey = partKeys.get(partKeys.size() - 1); try { - StatObjectResponse stat = - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(lastKey).build()); - long partSize = stat.size(); + long partSize = getPartSize(lastKey); if (bytesToRemoveRemaining >= partSize) { // The entire part is subsumed by the truncation; delete it and adjust remaining budget @@ -1495,7 +1576,8 @@ private void truncateManifestParts(UploadInfo uploadInfo, long bytesToRemove) { bytesToRemoveRemaining -= partSize; } else { // Only a fraction of this part is being removed; rewrite the remaining prefix - truncateSinglePart(lastKey, partSize, bytesToRemoveRemaining); + String newKey = truncateSinglePart(lastKey, partSize, bytesToRemoveRemaining); + partKeys.set(partKeys.size() - 1, newKey); bytesToRemoveRemaining = 0; } } catch (Exception e) { @@ -1507,28 +1589,38 @@ private void truncateManifestParts(UploadInfo uploadInfo, long bytesToRemove) { } /** - * Partially truncates an individual S3 part object by downloading its content and overwriting it - * in S3 with only the remaining byte prefix. + * Partially truncates an individual S3 part object by streaming only the remaining byte prefix + * into a new S3 object key with an updated size suffix, deleting the old part object. * * @param partKey S3 object key of the part to partially truncate * @param partSize Current total size of the part object * @param bytesToRemove Number of tail bytes to remove from this specific part + * @return The updated S3 part key with the new size suffix * @throws IOException If reading or re-uploading the truncated part fails */ - private void truncateSinglePart(String partKey, long partSize, long bytesToRemove) + private String truncateSinglePart(String partKey, long partSize, long bytesToRemove) throws IOException { - try (InputStream partStream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(partKey).build())) { - byte[] bytes = IOUtils.toByteArray(partStream); - int newLength = (int) (partSize - bytesToRemove); - byte[] remaining = Arrays.copyOf(bytes, newLength); + long newLength = partSize - bytesToRemove; + int underscoreIdx = partKey.lastIndexOf('_'); + String newPartKey = + underscoreIdx >= 0 ? partKey.substring(0, underscoreIdx + 1) + newLength : partKey; + + // Stream-based truncation without buffering entire part into JVM heap + try (InputStream partStream = S3Utils.getObjectInputStream(minioClient, bucket, partKey)) { + BoundedInputStream boundedStream = + BoundedInputStream.builder().setInputStream(partStream).setMaxCount(newLength).get(); minioClient.putObject( - PutObjectArgs.builder().bucket(bucket).object(partKey).stream( - new ByteArrayInputStream(remaining), (long) remaining.length, -1L) + PutObjectArgs.builder().bucket(bucket).object(newPartKey).stream( + boundedStream, newLength, -1L) .build()); } catch (Exception e) { throw new IOException("Failed to partially truncate S3 part object " + partKey, e); } + + if (!newPartKey.equals(partKey)) { + deleteObjectQuietly(partKey); + } + return newPartKey; } /** @@ -1543,14 +1635,14 @@ private void truncateSinglePart(String partKey, long partSize, long bytesToRemov private void truncateCompletedObject(UploadInfo uploadInfo, String objectKey, long newOffset) throws IOException { if (newOffset > 0) { - try (InputStream objStream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(objectKey).build())) { - byte[] remainingBytes = new byte[(int) newOffset]; - IOUtils.readFully(objStream, remainingBytes); - String chunkKey = buildChunkPartKey(uploadInfo.getId(), 1); + String chunkKey = buildChunkPartKey(uploadInfo.getId(), 1, newOffset); + // Stream-based truncation without buffering full completed object into JVM heap + try (InputStream objStream = S3Utils.getObjectInputStream(minioClient, bucket, objectKey)) { + BoundedInputStream boundedStream = + BoundedInputStream.builder().setInputStream(objStream).setMaxCount(newOffset).get(); minioClient.putObject( PutObjectArgs.builder().bucket(bucket).object(chunkKey).stream( - new ByteArrayInputStream(remainingBytes), (long) remainingBytes.length, -1L) + boundedStream, newOffset, -1L) .build()); uploadInfo.setUploadPartKeys(new ArrayList<>(Collections.singletonList(chunkKey))); } catch (Exception e) { @@ -1567,17 +1659,10 @@ void calculateAndSetOffset(UploadInfo info) { return; } String objectKey = getS3ObjectKey(info); - if (objectExists(objectKey)) { - try { - StatObjectResponse head = - minioClient.statObject( - StatObjectArgs.builder().bucket(bucket).object(objectKey).build()); - if (head != null) { - info.setOffset(head.size()); - return; - } - } catch (Exception ignored) { - } + long size = S3Utils.getObjectSize(minioClient, bucket, objectKey); + if (size >= 0) { + info.setOffset(size); + return; } List partKeys = @@ -1588,17 +1673,17 @@ void calculateAndSetOffset(UploadInfo info) { private List fetchExistingPartKeys(UploadId id) { String prefix = metadataPrefix + Objects.toString(id) + ".part."; - List partKeys = new ArrayList<>(); try { Iterable> results = minioClient.listObjects(ListObjectsArgs.builder().bucket(bucket).prefix(prefix).build()); - for (Result res : results) { - partKeys.add(res.get().objectName()); - } + return StreamSupport.stream(results.spliterator(), false) + .map(S3Utils::extractObjectName) + .filter(Objects::nonNull) + .sorted() + .collect(Collectors.toList()); } catch (Exception ignored) { + return Collections.emptyList(); } - Collections.sort(partKeys); - return partKeys; } private void deleteAllPartObjectsQuietly(UploadId id) { @@ -1607,77 +1692,12 @@ private void deleteAllPartObjectsQuietly(UploadId id) { } /** - * Deletes multiple S3 objects efficiently using batch Multi-Object Delete (POST /?delete), with - * automatic fallback to individual single-object DELETE requests if batch delete is unsupported - * or rejected by bucket IAM permissions. - * - *

    If batch delete fails once, it is permanently disabled for this storage service instance to - * avoid unnecessary failing S3 API round-trips. + * Quietly deletes multiple S3 objects, delegating to {@link S3Utils#deleteS3ObjectsQuietly}. * * @param keys The S3 object keys to delete */ void deleteS3ObjectsQuietly(Collection keys) { - if (keys == null || keys.isEmpty()) { - return; - } - - if (!isSupportsBatchDelete()) { - deleteObjectsIndividually(keys); - return; - } - - List objects = toDeleteRequestObjects(keys); - if (objects.isEmpty()) { - return; - } - - boolean batchFailed = false; - try { - Iterable> results = - minioClient.removeObjects( - RemoveObjectsArgs.builder().bucket(bucket).objects(objects).build()); - if (results != null) { - for (Result res : results) { - if (res != null) { - DeleteResult.Error error = res.get(); - if (error != null) { - log.debug( - "Batch delete reported error for key {}: {}; switching to individual delete", - error.objectName(), - error.message()); - batchFailed = true; - break; - } - } - } - } - } catch (Exception e) { - log.info( - "Batch multi-delete failed; permanently switching to individual delete: {}", - e.getMessage()); - batchFailed = true; - } - - if (batchFailed) { - setSupportsBatchDelete(false); - deleteObjectsIndividually(keys); - } - } - - private List toDeleteRequestObjects(Collection keys) { - List objects = new ArrayList<>(); - for (String key : keys) { - if (key != null) { - objects.add(new DeleteRequest.Object(key)); - } - } - return objects; - } - - private void deleteObjectsIndividually(Collection keys) { - for (String key : keys) { - deleteObjectQuietly(key); - } + S3Utils.deleteS3ObjectsQuietly(minioClient, bucket, keys); } /** @@ -1719,36 +1739,11 @@ void validateRemainingPartBudget(UploadInfo upload, int currentPartCount) } private boolean objectExists(String key) { - try { - minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(key).build()); - return true; - } catch (ErrorResponseException e) { - if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { - return false; - } - return false; - } catch (Exception e) { - return false; - } + return S3Utils.objectExists(minioClient, bucket, key); } private void deleteObjectQuietly(String key) { - if (key == null) { - return; - } - try { - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); - } catch (Exception e) { - log.debug("Failed to delete S3 object key {}", key, e); - } - } - - private String sanitizePrefix(String prefix) { - if (prefix == null || prefix.isEmpty()) { - return ""; - } - String result = prefix.startsWith("/") ? prefix.substring(1) : prefix; - return result.endsWith("/") ? result : result + "/"; + S3Utils.deleteS3ObjectQuietly(minioClient, bucket, key); } private String buildObjectKey(UploadId id) { @@ -1759,11 +1754,12 @@ private String buildMetadataKey(UploadId id) { return metadataPrefix + Objects.toString(id) + ".info"; } - private String buildChunkPartKey(UploadId id, int partNumber) { + private String buildChunkPartKey(UploadId id, int partNumber, long partSize) { return metadataPrefix + Objects.toString(id) + ".part." - + String.format("%05d-%s", partNumber, UUID.randomUUID().toString().substring(0, 8)); + + String.format( + "%05d-%s_%d", partNumber, UUID.randomUUID().toString().substring(0, 8), partSize); } private static class AppendResult { @@ -1852,7 +1848,7 @@ public InputStream nextElement() { } String key = partKeys.get(index++); try { - return minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(key).build()); + return S3Utils.getObjectInputStream(minioClient, bucket, key); } catch (Exception e) { throw new RuntimeException("Failed to read part " + key + " for streaming merge", e); } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java index 318a7a0b..4d5fed10 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3UploadLock.java @@ -1,24 +1,16 @@ package me.desair.tus.server.upload.s3; -import io.minio.GetObjectArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; -import io.minio.RemoveObjectArgs; -import io.minio.RemoveObjectsArgs; -import io.minio.Result; import io.minio.errors.ErrorResponseException; -import io.minio.messages.DeleteRequest; -import io.minio.messages.DeleteResult; import java.io.ByteArrayInputStream; import java.io.InputStream; -import java.util.List; +import java.util.Arrays; import java.util.Map; -import java.util.Objects; import java.util.concurrent.ScheduledExecutorService; -import java.util.stream.Collectors; -import java.util.stream.Stream; import me.desair.tus.server.upload.AbstractLeaseLock; import me.desair.tus.server.upload.LeaseData; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.util.LeaseDataJsonSerializer; import org.apache.commons.lang3.Strings; @@ -46,6 +38,7 @@ public class S3UploadLock extends AbstractLeaseLock { * daemon. * * @param leaseData The lease metadata + * @param uploadId The upload identifier associated with this lock * @param minioClient The MinIO client * @param bucket The S3 bucket * @param lockKey The S3 object key for the lock lease @@ -54,23 +47,25 @@ public class S3UploadLock extends AbstractLeaseLock { */ S3UploadLock( LeaseData leaseData, + UploadId uploadId, MinioClient minioClient, String bucket, String lockKey, String stopKey, - Map inputStreamMap) { - this(leaseData, minioClient, bucket, lockKey, stopKey, inputStreamMap, null); + Map inputStreamMap) { + this(leaseData, uploadId, minioClient, bucket, lockKey, stopKey, inputStreamMap, null); } S3UploadLock( LeaseData leaseData, + UploadId uploadId, MinioClient minioClient, String bucket, String lockKey, String stopKey, - Map inputStreamMap, + Map inputStreamMap, ScheduledExecutorService heartbeatExecutor) { - super(leaseData, inputStreamMap, heartbeatExecutor, "s3-lock-heartbeat"); + super(leaseData, uploadId, inputStreamMap, heartbeatExecutor); this.minioClient = minioClient; this.bucket = bucket; this.lockKey = lockKey; @@ -112,11 +107,15 @@ protected void doRenewLease() throws Exception { @Override protected void releaseLockResource() { - // Owner-Safe Lock Release: only delete .lock lease if it is still owned by this holder. - // If ownership was stolen by another node (e.g. heartbeat lost or execution ran too long), - // skip deletion entirely so we do not delete another node's lock or any stop signal - // potentially intended for that new holder. - if (lockKey != null && !doesLockOwnershipMatch(lockKey)) { + // Fast-path lock release: if the local lease is still well within its validity window + // (taking into account the 2000ms clock skew buffer), we know no other node could have + // evicted or acquired it. We can skip the expensive remote GET check, saving 1 network + // round trip on normal requests. + // Only if local execution ran beyond the lease TTL do we verify remote ownership to + // prevent deleting a successor node's lock. + if (lockKey != null + && getLeaseData().isExpired(System.currentTimeMillis(), -2_000L) + && !doesLockOwnershipMatch(lockKey)) { log.info( "Skipping deletion of S3 lock key {}: lock is currently held by another node", lockKey); return; @@ -129,101 +128,42 @@ protected void releaseLockResource() { } void deleteS3Objects(String firstKey, String secondKey) { - if (minioClient == null || bucket == null) { - return; - } - List keysToDelete = - Stream.of(firstKey, secondKey).filter(Objects::nonNull).collect(Collectors.toList()); - if (keysToDelete.isEmpty()) { - return; - } - - boolean batchFailed = false; - List objects = - keysToDelete.stream().map(DeleteRequest.Object::new).collect(Collectors.toList()); - - try { - // First attempt fast S3 Multi-Object Delete (POST /?delete) in a single round trip. - // Supported by MinIO and compliant S3 endpoints where bucket policies permit multi-delete. - Iterable> results = - minioClient.removeObjects( - RemoveObjectsArgs.builder().bucket(bucket).objects(objects).build()); - if (results != null) { - for (Result result : results) { - try { - if (result != null) { - DeleteResult.Error error = result.get(); - if (error != null) { - // S3 returned an error for an object (e.g. AccessDenied on multi-delete POST - // /?delete) - log.debug( - "Batch delete reported error for S3 object {}: {}; falling back to individual" - + " delete", - error.objectName(), - error.message()); - batchFailed = true; - break; - } - } - } catch (Exception e) { - log.debug( - "Exception iterating batch delete results; falling back to individual delete", e); - batchFailed = true; - break; - } - } - } - } catch (Exception e) { - log.debug( - "Batch multi-delete failed for keys {}; falling back to individual single-object DELETE", - keysToDelete, - e); - batchFailed = true; - } - - // Fall back to resilient individual single-object DELETE calls (DELETE /bucket/key). - // Standard AWS IAM policies often grant s3:DeleteObject exclusively on arn:aws:s3:::bucket/*, - // which allows individual object deletions while rejecting bucket-level multi-object delete. - if (batchFailed) { - for (String key : keysToDelete) { - deleteSingleObject(key); - } - } + S3Utils.deleteS3ObjectsQuietly(minioClient, bucket, Arrays.asList(firstKey, secondKey)); } void deleteSingleObject(String key) { - if (minioClient == null || bucket == null || key == null) { - return; - } - try { - minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); - } catch (Exception e) { - log.warn("Failed to delete S3 lock object key {} in bucket {}", key, bucket, e); - } + S3Utils.deleteS3ObjectQuietly(minioClient, bucket, key); } /** * Verifies that the S3 lock object is still owned by this lock holder. * + *

    Fail-closed split-brain protection: returns {@code true} only when the remote lock object + * exists and its holder ID matches this lock holder. On missing object, null data, S3 error, or + * network failure, returns {@code false} so a partitioned node cannot overwrite or delete a lock + * whose ownership it cannot confirm. + * * @param key The S3 lock object key - * @return {@code true} if the lock object is missing or owned by this holder; {@code false} if - * owned by a different holder or arguments are invalid + * @return {@code true} if the lock object exists and matches this holder's ID; {@code false} + * otherwise */ boolean doesLockOwnershipMatch(String key) { if (key == null || minioClient == null || bucket == null) { return false; } - try (InputStream stream = - minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(key).build())) { + try (InputStream stream = S3Utils.getObjectInputStream(minioClient, bucket, key)) { LeaseData remoteLock = LeaseDataJsonSerializer.deserialize(stream); - if (remoteLock != null && !Strings.CS.equals(remoteLock.getHolderId(), getHolderId())) { - return false; - } + return remoteLock != null && Strings.CS.equals(remoteLock.getHolderId(), getHolderId()); } catch (ErrorResponseException e) { - // Object is already gone or missing, safe to proceed + if (S3Utils.parseErrorResponse(e) == S3ErrorType.NO_SUCH_KEY) { + log.debug("S3 lock object {} does not exist, ownership cannot match", key); + } else { + log.warn("S3 error checking lock ownership for key {}: {}", key, e.getMessage()); + } + return false; } catch (Exception e) { - log.debug("Error checking lock ownership for S3 key {}", key, e); + log.warn("Error checking lock ownership for S3 key {}", key, e); + return false; } - return true; } } diff --git a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java index 2d0c2eaa..486462b1 100644 --- a/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java +++ b/src/main/java/me/desair/tus/server/upload/s3/S3Utils.java @@ -1,15 +1,242 @@ package me.desair.tus.server.upload.s3; +import io.minio.GetObjectArgs; +import io.minio.MinioClient; +import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; +import io.minio.Result; +import io.minio.StatObjectArgs; +import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteRequest; +import io.minio.messages.DeleteResult; import io.minio.messages.ErrorResponse; +import io.minio.messages.Item; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** Utility helper methods for parsing and evaluating MinIO S3 error responses. */ +/** Utility helper methods for interacting with MinIO and S3-compatible object storage backends. */ public final class S3Utils { + private static final Logger log = LoggerFactory.getLogger(S3Utils.class); + private S3Utils() { // Utility class } + /** + * Instantiates a pre-configured {@link MinioClient} with region fallback. + * + * @param endpoint S3 endpoint URL (e.g. "https://s3.amazonaws.com" or "http://localhost:9000") + * @param region S3 region name (defaults to "local" if null or empty) + * @param accessKey S3 access key / username + * @param secretKey S3 secret key / password + * @return Configured {@link MinioClient} instance + */ + public static MinioClient buildMinioClient( + String endpoint, String region, String accessKey, String secretKey) { + String effectiveRegion = (region != null && !region.isEmpty()) ? region : "local"; + return MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region(effectiveRegion) + .build(); + } + + /** + * Retrieves an {@link InputStream} for reading the contents of an S3 object. + * + * @param minioClient The {@link MinioClient} + * @param bucket The target bucket name + * @param objectKey The S3 object key to fetch + * @return {@link InputStream} representing the object content stream, or {@code null} if any + * argument is null + * @throws Exception If an error occurs while communicating with the S3 service + */ + public static InputStream getObjectInputStream( + MinioClient minioClient, String bucket, String objectKey) throws Exception { + if (minioClient == null || bucket == null || objectKey == null) { + return null; + } + return minioClient.getObject(GetObjectArgs.builder().bucket(bucket).object(objectKey).build()); + } + + /** + * Sanitizes an object key prefix, stripping leading slashes, collapsing trailing slashes, and + * ensuring exactly one trailing slash. + * + * @param prefix Raw prefix string + * @return Normalized prefix ending with a slash, or empty string if null, empty, or only slashes + */ + public static String sanitizePrefix(String prefix) { + if (prefix == null || prefix.trim().isEmpty()) { + return ""; + } + String result = prefix.trim(); + while (result.startsWith("/")) { + result = result.substring(1); + } + if (result.isEmpty()) { + return ""; + } + while (result.endsWith("/")) { + result = result.substring(0, result.length() - 1); + } + return result + "/"; + } + + /** + * Deletes a single S3 object, throwing an exception if the S3 request fails. + * + * @param minioClient The {@link MinioClient} + * @param bucket The target bucket name + * @param key The S3 object key to remove + * @throws Exception If an error occurs during object deletion + */ + public static void deleteS3Object(MinioClient minioClient, String bucket, String key) + throws Exception { + if (minioClient == null || bucket == null || key == null) { + return; + } + minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucket).object(key).build()); + } + + /** + * Quietly deletes a single S3 object without throwing exceptions, logging errors at debug level. + * + * @param minioClient The {@link MinioClient} + * @param bucket The target bucket name + * @param key The S3 object key to remove + */ + public static void deleteS3ObjectQuietly(MinioClient minioClient, String bucket, String key) { + try { + deleteS3Object(minioClient, bucket, key); + } catch (Exception e) { + log.debug("Failed to delete S3 object key {} in bucket {}: {}", key, bucket, e.getMessage()); + } + } + + /** + * Quietly deletes multiple S3 objects in a single batch request using S3 Multi-Object Delete. + * + *

    If batch multi-delete encounters an error (e.g. bucket policy restrictions or unsupported + * multi-delete), safely falls back to individual object deletions. + * + * @param minioClient The {@link MinioClient} + * @param bucket The target bucket name + * @param keys Collection of object keys to remove + */ + public static void deleteS3ObjectsQuietly( + MinioClient minioClient, String bucket, Collection keys) { + if (minioClient == null || bucket == null || keys == null || keys.isEmpty()) { + return; + } + List objects = + keys.stream() + .filter(Objects::nonNull) + .filter(k -> !k.trim().isEmpty()) + .map(DeleteRequest.Object::new) + .collect(Collectors.toList()); + if (objects.isEmpty()) { + return; + } + try { + Iterable> results = + minioClient.removeObjects( + RemoveObjectsArgs.builder().bucket(bucket).objects(objects).build()); + if (results != null) { + for (Result res : results) { + if (res != null) { + // Must iterate over lazy results to trigger execution in MinIO SDK + DeleteResult.Error error = res.get(); + if (error != null) { + throw new IOException( + "Batch multi-delete failed for object " + + error.objectName() + + ": " + + error.message()); + } + } + } + } + } catch (Exception e) { + log.debug( + "Batch multi-delete failed in bucket {}, falling back to individual deletes: {}", + bucket, + e.getMessage()); + for (String key : keys) { + deleteS3ObjectQuietly(minioClient, bucket, key); + } + } + } + + /** + * Extracts the object name from a MinIO {@link Result} of {@link Item}. + * + * @param result MinIO result item + * @return The object name, or {@code null} if result or item is null + * @throws RuntimeException If an error occurs retrieving the item from MinIO + */ + public static String extractObjectName(Result result) { + if (result == null) { + return null; + } + try { + Item item = result.get(); + return item != null ? item.objectName() : null; + } catch (Exception e) { + throw new RuntimeException("Failed to read S3 item result", e); + } + } + + /** + * Retrieves the size in bytes of an S3 object via {@code statObject}. + * + * @param minioClient The {@link MinioClient} + * @param bucket The target bucket name + * @param objectKey The S3 object key + * @return The object size in bytes, or -1L if the object does not exist or an error occurs + */ + public static long getObjectSize(MinioClient minioClient, String bucket, String objectKey) { + if (minioClient == null || bucket == null || objectKey == null) { + return -1L; + } + try { + StatObjectResponse stat = + minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(objectKey).build()); + return stat != null ? stat.size() : -1L; + } catch (Exception e) { + return -1L; + } + } + + /** + * Checks whether an S3 object exists in the specified bucket. + * + * @param minioClient The {@link MinioClient} + * @param bucket The target bucket name + * @param objectKey The S3 object key to check + * @return {@code true} if the object exists; {@code false} if it does not exist or an error + * occurs + */ + public static boolean objectExists(MinioClient minioClient, String bucket, String objectKey) { + if (minioClient == null || bucket == null || objectKey == null) { + return false; + } + try { + minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(objectKey).build()); + return true; + } catch (Exception e) { + return false; + } + } + /** * Parses an {@link ErrorResponseException} into a clean, strongly-typed {@link S3ErrorType}. * diff --git a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java index 6f6aa912..d6bf3fac 100644 --- a/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java +++ b/src/main/java/me/desair/tus/server/upload/util/AsyncChunkUploader.java @@ -245,10 +245,11 @@ public void abort() { } catch (Exception ignored) { } } - if (inFlightFile != null) { - FileUtils.deleteQuietly(inFlightFile); - inFlightFile = null; - } + // Note: Do NOT delete inFlightFile here. Ownership of inFlightFile belongs exclusively + // to the background worker thread's finally block in submitToSlot3(), avoiding race conditions + // where abort() deletes the file while the cloud SDK is still actively reading from it. + inFlightFile = null; + if (waitingChunkFile != null) { FileUtils.deleteQuietly(waitingChunkFile); waitingChunkFile = null; @@ -278,19 +279,28 @@ private void submitToSlot3( this.inFlightFile = tempFile; this.inFlightKey = preassignedKey; - // Wrap the upload action so temp file cleanup is guaranteed on the worker thread - this.inFlightUpload = - executor.submit( - () -> { - try { - uploadAction.upload(); - } finally { - // The worker owns tempFile once submitted; delete it immediately upon upload - // completion - FileUtils.deleteQuietly(tempFile); - } - return null; - }); + try { + // Wrap the upload action so temp file cleanup is guaranteed on the worker thread + this.inFlightUpload = + executor.submit( + () -> { + try { + uploadAction.upload(); + } finally { + // The worker owns tempFile once submitted; delete it immediately upon upload + // completion or interruption + FileUtils.deleteQuietly(tempFile); + } + return null; + }); + } catch (Throwable t) { + // If submitting to executor fails (e.g. RejectedExecutionException), delete tempFile + // immediately + FileUtils.deleteQuietly(tempFile); + this.inFlightFile = null; + this.inFlightKey = null; + throw t; + } } void checkAndConfirmInFlight() throws IOException { diff --git a/src/main/java/me/desair/tus/server/util/HttpChunkedEncodingInputStream.java b/src/main/java/me/desair/tus/server/util/HttpChunkedEncodingInputStream.java index 43d42fc9..a12f56f1 100644 --- a/src/main/java/me/desair/tus/server/util/HttpChunkedEncodingInputStream.java +++ b/src/main/java/me/desair/tus/server/util/HttpChunkedEncodingInputStream.java @@ -234,8 +234,12 @@ private void parseTrailerHeaders() throws IOException { } /** - * Upon close, this reads the remainder of the chunked message, leaving the underlying socket at a - * position to start reading the next response without scanning. + * Closes the stream and the underlying input stream immediately. + * + *

    When an upload stream is closed (for example, upon client disconnect, interruption, or + * cancellation), attempting to drain remaining chunks over the socket until EOF causes thread + * hangs or wasted network I/O. Closing the underlying stream immediately aborts the connection + * cleanly. * * @throws IOException If an IO problem occurs. */ @@ -243,37 +247,14 @@ private void parseTrailerHeaders() throws IOException { public void close() throws IOException { if (!closed) { try { - if (!eof) { - exhaustInputStream(); - } - } finally { eof = true; + in.close(); + } finally { closed = true; } } } - /** - * Exhaust our input stream, reading until EOF has been encountered. - * - *

    Note that this function is intended as a non-public utility. This is a little weird, but it - * seemed silly to make a utility class for this one function, so instead it is just static and - * shared that way. - * - * @throws IOException If an IO problem occurs - */ - private void exhaustInputStream() throws IOException { - // read and discard the remainder of the message - byte[] buffer = new byte[1024]; - - log.trace("Clearing underlying input stream, this is what was left:"); - while (in.read(buffer) >= 0) { - if (log.isTraceEnabled()) { - log.trace(new String(buffer, StandardCharsets.UTF_8)); - } - } - } - private List> parseHeaders(InputStream is, Charset charset) throws IOException { List> headers = new LinkedList<>(); diff --git a/src/main/java/me/desair/tus/server/util/LeaseDataJsonSerializer.java b/src/main/java/me/desair/tus/server/util/LeaseDataJsonSerializer.java index 388d995a..661b2a54 100644 --- a/src/main/java/me/desair/tus/server/util/LeaseDataJsonSerializer.java +++ b/src/main/java/me/desair/tus/server/util/LeaseDataJsonSerializer.java @@ -10,6 +10,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import me.desair.tus.server.upload.LeaseData; +import me.desair.tus.server.upload.disk.DiskUtils; /** * Utility class responsible for serializing and deserializing {@link LeaseData} objects to and from @@ -98,9 +99,9 @@ public static void serializeToFile(LeaseData data, File file) throws IOException */ public static void serializeToPath(LeaseData data, Path path) throws IOException { if (data != null && path != null) { - try (Utils.TempPath tempPath = new Utils.TempPath(path)) { + try (DiskUtils.TempPath tempPath = new DiskUtils.TempPath(path)) { serializeToFile(data, tempPath.getPath().toFile()); - Utils.atomicMove(tempPath.getPath(), path); + DiskUtils.atomicMove(tempPath.getPath(), path); } } } diff --git a/src/main/java/me/desair/tus/server/util/UploadInfoJsonSerializer.java b/src/main/java/me/desair/tus/server/util/UploadInfoJsonSerializer.java index f3a02bdc..d83c0fed 100644 --- a/src/main/java/me/desair/tus/server/util/UploadInfoJsonSerializer.java +++ b/src/main/java/me/desair/tus/server/util/UploadInfoJsonSerializer.java @@ -17,6 +17,7 @@ import java.nio.file.Path; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; +import me.desair.tus.server.upload.disk.DiskUtils; /** * Utility class responsible for serializing and deserializing {@link UploadInfo} and other domain @@ -132,9 +133,9 @@ public static void serializeToFile(Object object, File file) throws IOException */ public static void serializeToPath(Object object, Path path) throws IOException { if (object != null && path != null) { - try (Utils.TempPath tempPath = new Utils.TempPath(path)) { + try (DiskUtils.TempPath tempPath = new DiskUtils.TempPath(path)) { serializeToFile(object, tempPath.getPath().toFile()); - Utils.atomicMove(tempPath.getPath(), path); + DiskUtils.atomicMove(tempPath.getPath(), path); } } } diff --git a/src/main/java/me/desair/tus/server/util/Utils.java b/src/main/java/me/desair/tus/server/util/Utils.java index aaeb6ed4..b0eec0ea 100644 --- a/src/main/java/me/desair/tus/server/util/Utils.java +++ b/src/main/java/me/desair/tus/server/util/Utils.java @@ -1,32 +1,24 @@ package me.desair.tus.server.util; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; -import static java.nio.file.StandardOpenOption.WRITE; - import jakarta.servlet.http.HttpServletRequest; -import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.ObjectOutput; -import java.io.ObjectOutputStream; -import java.io.OutputStream; import java.io.Serializable; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.EnumSet; import java.util.LinkedList; import java.util.List; import java.util.Objects; -import java.util.UUID; -import java.util.concurrent.Executors; +import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import me.desair.tus.server.HttpHeader; import me.desair.tus.server.HttpMethod; @@ -36,7 +28,7 @@ import me.desair.tus.server.upload.UploadInfo; import me.desair.tus.server.upload.UploadLockingService; import me.desair.tus.server.upload.UploadStorageService; -import org.apache.commons.io.serialization.ValidatingObjectInputStream; +import me.desair.tus.server.upload.disk.DiskUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Strings; import org.slf4j.Logger; @@ -46,8 +38,6 @@ public class Utils { private static final Logger log = LoggerFactory.getLogger(Utils.class); - private static final int LOCK_FILE_RETRY_COUNT = 3; - private static final long LOCK_FILE_SLEEP_TIME = 500; private static final Pattern CHECKSUM_VALUE_PATTERN = Pattern.compile("^[a-zA-Z0-9+/=\\-_]+$"); private Utils() { @@ -102,19 +92,7 @@ public static List parseConcatenationIDsFromHeader(String uploadConcatVa * @throws IOException If file access fails */ public static T readSerializable(Path path, Class clazz) throws IOException { - T info = null; - if (path != null && Files.exists(path)) { - try (InputStream is = Files.newInputStream(path); - ValidatingObjectInputStream ois = new ValidatingObjectInputStream(is)) { - ois.accept("java.lang.*", "java.util.*", "me.desair.tus.server.*"); - info = clazz.cast(ois.readObject()); - } catch (ClassNotFoundException | java.io.EOFException | java.io.StreamCorruptedException e) { - // File may be corrupted due to unexpected server shutdown - log.warn("Unable to read serializable file {}: {}", path, e.getMessage()); - info = null; - } - } - return info; + return DiskUtils.readSerializable(path, clazz); } /** @@ -136,46 +114,6 @@ public static boolean deletePathQuietly(Path path) { } } - /** - * Resolves a unique temporary sibling path for a target destination file path. - * - * @param targetPath The destination file path - * @return A temporary sibling path with a unique UUID suffix, or null if targetPath is null - */ - public static Path createTempSiblingPath(Path targetPath) { - if (targetPath == null) { - return null; - } - Path parent = targetPath.getParent(); - String tempFileName = targetPath.getFileName().toString() + ".tmp." + UUID.randomUUID(); - return parent != null ? parent.resolve(tempFileName) : Paths.get(tempFileName); - } - - /** - * Creates an {@link AutoCloseable} {@link TempPath} that generates a unique temporary sibling - * path and automatically deletes it upon closing if it still exists. - * - * @param targetPath The destination file path - * @return An AutoCloseable TempPath instance - */ - public static TempPath createTempSibling(Path targetPath) { - return new TempPath(targetPath); - } - - /** - * Atomically moves a source file to a destination path, replacing any existing destination file. - * - * @param source The source file path to move - * @param destination The target destination file path - * @throws IOException If moving the file fails - */ - public static void atomicMove(Path source, Path destination) throws IOException { - if (source != null && destination != null) { - Files.move( - source, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } - } - /** * Writes a serializable object to a file on disk atomically via a temporary file rename. * @@ -184,57 +122,15 @@ public static void atomicMove(Path source, Path destination) throws IOException * @throws IOException If file access or move fails */ public static void writeSerializable(Serializable object, Path path) throws IOException { - if (path != null && object != null) { - try (TempPath tempPath = new TempPath(path)) { - try (OutputStream os = - Files.newOutputStream(tempPath.getPath(), WRITE, CREATE, TRUNCATE_EXISTING); - OutputStream buffer = new BufferedOutputStream(os); - ObjectOutput output = new ObjectOutputStream(buffer)) { - - output.writeObject(object); - } - atomicMove(tempPath.getPath(), path); - } - } - } - - /** - * AutoCloseable temporary sibling path that automatically cleans up (deletes) the temporary file - * upon closing if it still exists. - */ - public static class TempPath implements AutoCloseable { - private final Path path; - - /** - * Constructs a TempPath resolving a unique sibling path for the given target path. - * - * @param targetPath The destination file path - */ - public TempPath(Path targetPath) { - this.path = createTempSiblingPath(targetPath); - } - - /** - * Returns the underlying temporary sibling {@link Path}. - * - * @return The temporary path, or null if targetPath was null - */ - public Path getPath() { - return path; - } - - @Override - public void close() { - deletePathQuietly(path); - } + DiskUtils.writeSerializable(object, path); } public static FileLock lockFileExclusively(FileChannel channel) throws IOException { - return lockFile(channel, false); + return DiskUtils.lockFileExclusively(channel); } public static FileLock lockFileShared(FileChannel channel) throws IOException { - return lockFile(channel, true); + return DiskUtils.lockFileShared(channel); } /** @@ -252,22 +148,6 @@ public static void sleep(long sleepTimeMillis) { } } - private static FileLock lockFile(FileChannel channel, boolean shared) throws IOException { - int i = 0; - FileLock lock = null; - do { - if (i > 0) { - sleep(LOCK_FILE_SLEEP_TIME); - } - - lock = channel.tryLock(0L, Long.MAX_VALUE, shared); - - i++; - } while (lock == null && i < LOCK_FILE_RETRY_COUNT); - - return lock; - } - /** Helper class to store parsed checksum header information. */ public static class ChecksumInfo { private final ChecksumAlgorithm algorithm; @@ -563,12 +443,38 @@ public static String getUploadUriOnCreation( * @return A new single-thread ScheduledExecutorService */ public static ScheduledExecutorService createScheduledDaemonExecutor(String threadName) { - return Executors.newSingleThreadScheduledExecutor( - runnable -> { - Thread thread = new Thread(runnable, threadName); - thread.setDaemon(true); - return thread; - }); + return createScheduledDaemonExecutor(threadName, 1); + } + + /** + * Creates a scheduled thread pool executor with the specified core pool size and daemon threads. + * Enables immediate task removal upon cancellation to prevent memory buildup in long-running + * services. + * + * @param threadName The name prefix for the background daemon threads + * @param corePoolSize The number of threads to keep in the pool + * @return A new ScheduledExecutorService + */ + public static ScheduledExecutorService createScheduledDaemonExecutor( + String threadName, int corePoolSize) { + int poolSize = Math.max(1, corePoolSize); + ScheduledThreadPoolExecutor executor = + new ScheduledThreadPoolExecutor( + poolSize, + new ThreadFactory() { + private final AtomicInteger threadCounter = new AtomicInteger(1); + + @Override + public Thread newThread(Runnable runnable) { + String name = + poolSize > 1 ? threadName + "-" + threadCounter.getAndIncrement() : threadName; + Thread thread = new Thread(runnable, name); + thread.setDaemon(true); + return thread; + } + }); + executor.setRemoveOnCancelPolicy(true); + return executor; } /** @@ -584,7 +490,29 @@ public static ScheduledExecutorService createScheduledDaemonExecutor(String thre */ public static ScheduledExecutorService scheduleWatchdog( String threadName, Runnable task, long initialDelay, long period, TimeUnit unit) { - ScheduledExecutorService executor = createScheduledDaemonExecutor(threadName); + return scheduleWatchdog(threadName, 1, task, initialDelay, period, unit); + } + + /** + * Creates a multi-threaded daemon scheduled executor and immediately schedules a task to run + * periodically at a fixed rate. + * + * @param threadName The name for the background daemon thread + * @param corePoolSize The number of threads in the executor pool + * @param task The task to execute periodically + * @param initialDelay The initial delay before the first execution + * @param period The period between successive executions + * @param unit The time unit of the initialDelay and period parameters + * @return The created ScheduledExecutorService + */ + public static ScheduledExecutorService scheduleWatchdog( + String threadName, + int corePoolSize, + Runnable task, + long initialDelay, + long period, + TimeUnit unit) { + ScheduledExecutorService executor = createScheduledDaemonExecutor(threadName, corePoolSize); if (period > 0 && task != null) { executor.scheduleAtFixedRate(task, initialDelay, period, unit); } @@ -592,12 +520,12 @@ public static ScheduledExecutorService scheduleWatchdog( } /** - * Safely shuts down a ScheduledExecutorService using {@link - * ScheduledExecutorService#shutdownNow()}. + * Safely shuts down an {@link ExecutorService} using {@link ExecutorService#shutdownNow()}. + * Handles null or already shut down executors safely as a no-op. * - * @param executor The ScheduledExecutorService to shut down + * @param executor The ExecutorService to shut down, or null */ - public static void shutdownExecutor(ScheduledExecutorService executor) { + public static void shutdownExecutor(ExecutorService executor) { if (executor != null && !executor.isShutdown()) { try { executor.shutdownNow(); @@ -647,21 +575,6 @@ public static void interruptStream(java.io.InputStream inputStream) { } } - /** - * Safely interrupts and stops a thread, catching and logging any exceptions that occur. - * - * @param thread The Thread to interrupt - */ - public static void interruptThread(Thread thread) { - if (thread != null) { - try { - thread.interrupt(); - } catch (Exception e) { - log.debug("Error interrupting thread {}: {}", thread.getName(), e.getMessage()); - } - } - } - /** * Ensures that the specified directory exists, creating it and any necessary parent directories. * If the directory already exists, this method is a safe no-op. If the path is null, this method diff --git a/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java b/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java index 92909a7a..8573b722 100644 --- a/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java +++ b/src/test/java/me/desair/tus/server/AbstractITTusFileUploadService.java @@ -28,6 +28,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.TusException; import me.desair.tus.server.exception.UploadNotFoundException; import me.desair.tus.server.upload.UploadId; @@ -2844,6 +2845,74 @@ public void testDownloadFailsWhenParentBytesDeletedOutOfBandWithDeduplicationEna assertResponseStatus(HttpServletResponse.SC_NOT_FOUND); } + @Test + public void testChecksumIndexChildRegistrationEndToEnd() throws Exception { + tusFileUploadService.withUploadDeduplication(true); + + String content = "Test Child Index Registration " + UUID.randomUUID(); + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + String sha1Base64 = + org.apache.commons.codec.binary.Base64.encodeBase64String( + org.apache.commons.codec.digest.DigestUtils.sha1(contentBytes)); + + // 1. Create and complete parent upload + reset(); + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, contentBytes.length); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_CREATED); + String parentUri = servletResponse.getHeader(HttpHeader.LOCATION); + + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(parentUri); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, "application/offset+octet-stream"); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, 0); + servletRequest.addHeader(HttpHeader.UPLOAD_CHECKSUM, "sha1 " + sha1Base64); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.setContent(contentBytes); + UploadInfo parentInfo = + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_NO_CONTENT); + assertNotNull(parentInfo); + assertNull(parentInfo.getDuplicatesUploadId()); + + // 2. Create and complete child upload + reset(); + servletRequest.setMethod("POST"); + servletRequest.setRequestURI(UPLOAD_URI); + servletRequest.addHeader(HttpHeader.UPLOAD_LENGTH, contentBytes.length); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_CREATED); + String childUri = servletResponse.getHeader(HttpHeader.LOCATION); + + reset(); + servletRequest.setMethod("PATCH"); + servletRequest.setRequestURI(childUri); + servletRequest.addHeader(HttpHeader.CONTENT_TYPE, "application/offset+octet-stream"); + servletRequest.addHeader(HttpHeader.UPLOAD_OFFSET, 0); + servletRequest.addHeader(HttpHeader.UPLOAD_CHECKSUM, "sha1 " + sha1Base64); + servletRequest.addHeader(HttpHeader.TUS_RESUMABLE, "1.0.0"); + servletRequest.setContent(contentBytes); + UploadInfo childInfo = tusFileUploadService.process(servletRequest, servletResponse, OWNER_KEY); + assertResponseStatus(HttpServletResponse.SC_NO_CONTENT); + assertNotNull(childInfo); + assertEquals(parentInfo.getId(), childInfo.getDuplicatesUploadId()); + + // 3. Verify getUploadInfoByChecksum retrieves the parent + UploadInfo found = + tusFileUploadService + .getUploadStorageService() + .getUploadInfoByChecksum( + org.apache.commons.codec.digest.DigestUtils.sha1Hex(contentBytes), + ChecksumAlgorithm.SHA1); + assertNotNull(found); + assertEquals(parentInfo.getId(), found.getId()); + } + @Test public void testDownloadSucceedsForChildrenWhenParentBytesDeletedOutOfBandWithDeduplicationDisabled() diff --git a/src/test/java/me/desair/tus/server/CoverageGapTest.java b/src/test/java/me/desair/tus/server/CoverageGapTest.java index db42559a..8ddde36f 100644 --- a/src/test/java/me/desair/tus/server/CoverageGapTest.java +++ b/src/test/java/me/desair/tus/server/CoverageGapTest.java @@ -832,19 +832,23 @@ public void testDiskStorageServiceChecksumBase64EdgeCase() throws Exception { @Test public void testDiskStorageServiceUnsafePathComponent() throws Exception { - DiskStorageService storage = new DiskStorageService("/tmp"); - java.lang.reflect.Method isSafe = - DiskStorageService.class.getDeclaredMethod("isSafePathComponent", String.class); - isSafe.setAccessible(true); - - assertThat((Boolean) isSafe.invoke(storage, "safe-name"), is(true)); - assertThat((Boolean) isSafe.invoke(storage, "path/with/slash"), is(false)); - assertThat((Boolean) isSafe.invoke(storage, "path\\with\\backslash"), is(false)); - assertThat((Boolean) isSafe.invoke(storage, "../dotdot"), is(false)); - assertThat((Boolean) isSafe.invoke(storage, "question?mark"), is(false)); - assertThat((Boolean) isSafe.invoke(storage, "asterisk*file"), is(false)); - assertThat((Boolean) isSafe.invoke(storage, ""), is(false)); - assertThat((Boolean) isSafe.invoke(storage, (String) null), is(false)); + assertThat( + me.desair.tus.server.upload.disk.DiskUtils.isSafePathComponent("safe-name"), is(true)); + assertThat( + me.desair.tus.server.upload.disk.DiskUtils.isSafePathComponent("path/with/slash"), + is(false)); + assertThat( + me.desair.tus.server.upload.disk.DiskUtils.isSafePathComponent("path\\with\\backslash"), + is(false)); + assertThat( + me.desair.tus.server.upload.disk.DiskUtils.isSafePathComponent("../dotdot"), is(false)); + assertThat( + me.desair.tus.server.upload.disk.DiskUtils.isSafePathComponent("question?mark"), is(false)); + assertThat( + me.desair.tus.server.upload.disk.DiskUtils.isSafePathComponent("asterisk*file"), is(false)); + assertThat(me.desair.tus.server.upload.disk.DiskUtils.isSafePathComponent(""), is(false)); + assertThat( + me.desair.tus.server.upload.disk.DiskUtils.isSafePathComponent((String) null), is(false)); } @Test diff --git a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java index 25e2334c..e4588abd 100644 --- a/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java +++ b/src/test/java/me/desair/tus/server/TusFileUploadServiceTest.java @@ -25,6 +25,8 @@ import me.desair.tus.server.util.TusServletRequest; import me.desair.tus.server.util.TusServletResponse; import org.junit.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; public class TusFileUploadServiceTest { @@ -700,7 +702,7 @@ public void testAcquireUploadLockWithConfiguredRetries() throws Exception { UploadLock lock = service.acquireUploadLock(HttpMethod.HEAD, "/files/test"); assertNotNull(lock); - verify(mockLockingService, times(2)).requestLockRelease("/files/test"); + verify(mockLockingService, times(1)).requestLockRelease("/files/test"); } @Test @@ -984,7 +986,7 @@ public void testProcessCleansUpLockOnUploadCompletion() throws Exception { service.process(mockReq, mockResp, "owner"); - verify(mockLockingService).cleanupLock(testId); + verify(mockLock).close(); } @Test @@ -1050,4 +1052,23 @@ public void testDeleteUploadTerminatesUpload() throws Exception { verify(mockStorage).terminateUpload(info); } + + @Test + public void testProcessTusExceptionOverloadDelegates() throws Exception { + TusFileUploadService service = new TusFileUploadService(); + MockHttpServletRequest servletRequest = new MockHttpServletRequest(); + MockHttpServletResponse servletResponse = new MockHttpServletResponse(); + TusServletRequest request = new TusServletRequest(servletRequest); + TusServletResponse response = new TusServletResponse(servletResponse); + + service.processTusException( + HttpMethod.PATCH, + request, + response, + "owner", + new me.desair.tus.server.exception.TusException(HttpServletResponse.SC_BAD_REQUEST, "test"), + ProtocolVersion.TUS_1_0_0); + + assertEquals(HttpServletResponse.SC_BAD_REQUEST, servletResponse.getStatus()); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java index 2d6efd3c..547239ce 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhAppendPatchRequestHandlerTest.java @@ -165,13 +165,12 @@ public void testProcessCallFiveParameterMethodAndAppendReturnsNull() throws Exce request.setRequestURI("/files/append-id"); request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); - request.addHeader(HttpHeader.CONTENT_LENGTH, "4"); - request.setContent("data".getBytes()); + request.setContent(new byte[0]); UploadInfo info = new UploadInfo(); info.setId(new UploadId("append-id")); info.setOffset(1000L); - info.setLength(1004L); + info.setLength(1000L); when(storageService.getUploadInfo("/files/append-id", "owner")).thenReturn(info); // storageService.append returns null @@ -432,4 +431,56 @@ public void testProcessInterruptedByLockContention() throws Exception { assertThat(response.getStatus(), is(204)); assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("1005")); } + + @Test + public void testProcessInterruptedByLockContentionWhenDeferredLength() throws Exception { + request.setMethod("PATCH"); + request.setRequestURI("/files/append-id"); + request.addHeader(HttpHeader.CONTENT_TYPE, HttpHeader.CONTENT_TYPE_PARTIAL_UPLOAD); + request.addHeader(HttpHeader.UPLOAD_OFFSET, "1000"); + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.setContent("hello world".getBytes()); + + UploadInfo initial = new UploadInfo(); + initial.setId(new UploadId("append-id")); + initial.setOffset(1000L); + initial.setLength(null); // Deferred length initially + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("append-id")); + refreshed.setOffset(1005L); + refreshed.setLength(1011L); // Tentative length from line 66 before interruption + + when(storageService.getUploadInfo("/files/append-id", "owner")) + .thenReturn(initial) + .thenReturn(refreshed); + + when(storageService.append(eq(initial), any(InputStream.class))) + .thenAnswer( + inv -> { + Object s = inv.getArgument(1); + if (s instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) s).interrupt(); + } + throw new IOException("Stream interrupted by locking service contention"); + }); + + handler.process( + HttpMethod.PATCH, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(204)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("1005")); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadInfo.class); + verify(storageService, org.mockito.Mockito.atLeastOnce()).update(captor.capture()); + // The last update before returning must reset length back to null for deferred upload resume + org.junit.Assert.assertNull(captor.getValue().getLength()); + } } diff --git a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java index 76215ddd..2e497a66 100644 --- a/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java +++ b/src/test/java/me/desair/tus/server/rufh/handler/RufhCreationPostRequestHandlerTest.java @@ -690,4 +690,54 @@ public void testProcessCreationInterruptedByLockContention() throws Exception { assertThat(response.getStatus(), is(201)); assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("13")); } + + @Test + public void testProcessCreationInterruptedByLockContentionWhenDeferredLength() throws Exception { + request.setMethod("POST"); + request.setRequestURI("/files"); + // Deferred length: Upload-Length header is omitted; Upload-Complete is set to ?1 with body + request.addHeader(HttpHeader.UPLOAD_COMPLETE, "?1"); + request.setContent("creation body".getBytes()); + + UploadInfo info = new UploadInfo(); + info.setId(new UploadId("creation-id")); + info.setLength(13L); // Tentatively set length from contentLength + info.setOffset(0L); + + UploadInfo refreshed = new UploadInfo(); + refreshed.setId(new UploadId("creation-id")); + refreshed.setLength(13L); + refreshed.setOffset(5L); + + when(storageService.create(any(UploadInfo.class), nullable(String.class))).thenReturn(info); + when(storageService.getUploadInfo("/files/creation-id", "owner")).thenReturn(refreshed); + + when(storageService.append(eq(info), any(InputStream.class))) + .thenAnswer( + inv -> { + Object s = inv.getArgument(1); + if (s instanceof InterruptibleInputStream) { + ((InterruptibleInputStream) s).interrupt(); + } + throw new IOException("Stream interrupted by locking service contention"); + }); + + handler.process( + HttpMethod.POST, + new TusServletRequest(request), + new TusServletResponse(response), + storageService, + lockingService, + "owner", + null); + + assertThat(response.getStatus(), is(201)); + assertThat(response.getHeader(HttpHeader.UPLOAD_OFFSET), is("5")); + assertThat(response.getHeader(HttpHeader.UPLOAD_COMPLETE), is("?0")); + + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadInfo.class); + verify(storageService, org.mockito.Mockito.atLeastOnce()).update(captor.capture()); + // The update on interruption must reset length to null so deferred upload can be resumed + org.junit.Assert.assertNull(captor.getValue().getLength()); + } } diff --git a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java index bc7715c3..ee35579b 100644 --- a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockTest.java @@ -4,18 +4,22 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import me.desair.tus.server.util.InterruptibleInputStream; import org.junit.Test; public class AbstractLeaseLockTest { @@ -26,17 +30,15 @@ private static class TestLeaseLock extends AbstractLeaseLock { public TestLeaseLock( LeaseData leaseData, - Map activeInputStreams, - String watchdogThreadName) { - super(leaseData, activeInputStreams, watchdogThreadName); + UploadId uploadId, + Map activeInputStreams, + ScheduledExecutorService heartbeatExecutor) { + super(leaseData, uploadId, activeInputStreams, heartbeatExecutor); } public TestLeaseLock( - LeaseData leaseData, - Map activeInputStreams, - ScheduledExecutorService heartbeatExecutor, - String watchdogThreadName) { - super(leaseData, activeInputStreams, heartbeatExecutor, watchdogThreadName); + LeaseData leaseData, UploadId uploadId, Map activeInputStreams) { + this(leaseData, uploadId, activeInputStreams, null); } @Override @@ -52,11 +54,13 @@ protected void releaseLockResource() { @Test public void testGettersAndSetters() { + UploadId uploadId = new UploadId("upload-1"); LeaseData leaseData = new LeaseData("holder-1", "/files/upload-1", 30000L, 100000L, 50000L, "/path", "/stop"); - TestLeaseLock lock = new TestLeaseLock(leaseData, null, null); + TestLeaseLock lock = new TestLeaseLock(leaseData, uploadId, null, null); assertThat(lock.getHolderId(), is("holder-1")); + assertThat(lock.getUploadId(), is(uploadId)); assertThat(lock.getRequestUri(), is("/files/upload-1")); assertThat(lock.getUploadUri(), is("/files/upload-1")); assertThat(lock.getLeaseDurationMs(), is(30000L)); @@ -69,13 +73,19 @@ public void testGettersAndSetters() { } @Test - public void testActiveLockConstructorWithAutoWatchdog() throws Exception { - Map activeStreams = new ConcurrentHashMap<>(); - activeStreams.put("/files/upload-3", new ByteArrayInputStream("test".getBytes())); + public void testActiveLockConstructorWithExecutor() throws Exception { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + when(mockExecutor.scheduleWithFixedDelay(any(), anyLong(), anyLong(), any())) + .thenReturn(mockFuture); + + UploadId uploadId = new UploadId("upload-3"); + Map activeStreams = new ConcurrentHashMap<>(); + activeStreams.put(uploadId, new ByteArrayInputStream("test".getBytes())); LeaseData leaseData = new LeaseData("holder-3", "/files/upload-3", 10000L, System.currentTimeMillis() + 10000L); - TestLeaseLock lock = new TestLeaseLock(leaseData, activeStreams, "test-watchdog"); + TestLeaseLock lock = new TestLeaseLock(leaseData, uploadId, activeStreams, mockExecutor); assertThat(lock.getHolderId(), is("holder-3")); assertThat(lock.getLeaseDurationMs(), is(10000L)); @@ -91,69 +101,61 @@ public void testActiveLockConstructorWithAutoWatchdog() throws Exception { lock.release(); assertTrue(lock.released.get()); - assertFalse(activeStreams.containsKey("/files/upload-3")); - } - - @Test - public void testActiveLockConstructorWithCustomExecutor() { - ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); - Map activeStreams = new ConcurrentHashMap<>(); - - LeaseData leaseData = - new LeaseData("holder-4", "/files/upload-4", 10000L, System.currentTimeMillis() + 10000L); - TestLeaseLock lock = new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog"); - - assertNotNull(lock); - lock.close(); - - verify(mockExecutor).shutdownNow(); - assertTrue(lock.released.get()); + assertFalse(activeStreams.containsKey(uploadId)); + verify(mockFuture).cancel(true); } @Test public void testActiveLockConstructorWithZeroLeaseDurationDoesNotSchedule() { + UploadId uploadId = new UploadId("upload-5"); LeaseData leaseData = new LeaseData("holder-5", "/files/upload-5", 0L, 0L); - TestLeaseLock lock = new TestLeaseLock(leaseData, null, "test-watchdog"); + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + TestLeaseLock lock = new TestLeaseLock(leaseData, uploadId, null, mockExecutor); lock.close(); assertTrue(lock.released.get()); + org.mockito.Mockito.verifyNoInteractions(mockExecutor); LeaseData leaseData2 = new LeaseData("holder-5b", "/files/upload-5b", 10000L, 10000L); - TestLeaseLock lockNullWatchdog = new TestLeaseLock(leaseData2, null, null); - lockNullWatchdog.close(); - assertTrue(lockNullWatchdog.released.get()); + TestLeaseLock lockNullExecutor = new TestLeaseLock(leaseData2, uploadId, null, null); + lockNullExecutor.close(); + assertTrue(lockNullExecutor.released.get()); } @Test - public void testCloseWithNullStreamsOrNullUri() { + public void testCloseWithNullStreamsOrNullUploadId() { + UploadId uploadId = new UploadId("upload-6"); LeaseData leaseData = new LeaseData("holder-6", "/files/upload-6", 10000L, 10000L); - TestLeaseLock lockNullStreams = new TestLeaseLock(leaseData, null, "test-watchdog"); + TestLeaseLock lockNullStreams = new TestLeaseLock(leaseData, uploadId, null); lockNullStreams.close(); assertTrue(lockNullStreams.released.get()); - Map activeStreams = new ConcurrentHashMap<>(); - LeaseData leaseDataNullUri = new LeaseData("holder-7", null, 10000L, 10000L); - TestLeaseLock lockNullUri = new TestLeaseLock(leaseDataNullUri, activeStreams, "test-watchdog"); - lockNullUri.close(); - assertTrue(lockNullUri.released.get()); + Map activeStreams = new ConcurrentHashMap<>(); + LeaseData leaseDataNullId = new LeaseData("holder-7", null, 10000L, 10000L); + TestLeaseLock lockNullUploadId = new TestLeaseLock(leaseDataNullId, null, activeStreams); + lockNullUploadId.close(); + assertTrue(lockNullUploadId.released.get()); } @Test - public void testRenewLeaseFailureInterruptsActiveStreamAndShutsDownExecutor() { + public void testRenewLeaseFailureInterruptsActiveStreamAndCancelsHeartbeat() { ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); - Map activeStreams = new ConcurrentHashMap<>(); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + when(mockExecutor.scheduleWithFixedDelay(any(), anyLong(), anyLong(), any())) + .thenReturn(mockFuture); + UploadId uploadId = new UploadId("upload-failing"); + Map activeStreams = new ConcurrentHashMap<>(); - me.desair.tus.server.util.InterruptibleInputStream stream = - new me.desair.tus.server.util.InterruptibleInputStream( - new ByteArrayInputStream("test".getBytes())); - activeStreams.put("/files/upload-failing", stream); + InterruptibleInputStream stream = + new InterruptibleInputStream(new ByteArrayInputStream("test".getBytes())); + activeStreams.put(uploadId, stream); LeaseData leaseData = new LeaseData( "holder-fail", "/files/upload-failing", 10000L, System.currentTimeMillis() - 1000L); TestLeaseLock lock = - new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog") { + new TestLeaseLock(leaseData, uploadId, activeStreams, mockExecutor) { @Override protected void doRenewLease() { throw new RuntimeException("Simulated S3 network failure during lease renewal"); @@ -161,29 +163,32 @@ protected void doRenewLease() { }; // Trigger renewLease and verify that active streaming input stream was interrupted - // immediately to prevent un-locked writes, and heartbeat daemon stopped. + // immediately to prevent un-locked writes, and heartbeat daemon cancelled. lock.renewLease(); assertTrue(stream.isInterrupted()); - verify(mockExecutor).shutdownNow(); + verify(mockFuture).cancel(true); } @Test public void testRenewLeaseTransientFailureDoesNotInterruptActiveStreamIfLeaseValid() { ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); - Map activeStreams = new ConcurrentHashMap<>(); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + when(mockExecutor.scheduleWithFixedDelay(any(), anyLong(), anyLong(), any())) + .thenReturn(mockFuture); + UploadId uploadId = new UploadId("upload-transient"); + Map activeStreams = new ConcurrentHashMap<>(); - me.desair.tus.server.util.InterruptibleInputStream stream = - new me.desair.tus.server.util.InterruptibleInputStream( - new ByteArrayInputStream("test".getBytes())); - activeStreams.put("/files/upload-failing", stream); + InterruptibleInputStream stream = + new InterruptibleInputStream(new ByteArrayInputStream("test".getBytes())); + activeStreams.put(uploadId, stream); LeaseData leaseData = new LeaseData( "holder-fail", "/files/upload-failing", 10000L, System.currentTimeMillis() + 10000L); TestLeaseLock lock = - new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog") { + new TestLeaseLock(leaseData, uploadId, activeStreams, mockExecutor) { @Override protected void doRenewLease() { throw new RuntimeException("Simulated transient network glitch"); @@ -195,24 +200,28 @@ protected void doRenewLease() { org.junit.Assert.assertFalse(stream.isInterrupted()); org.mockito.Mockito.verify(mockExecutor, org.mockito.Mockito.never()).shutdownNow(); + org.mockito.Mockito.verify(mockFuture, org.mockito.Mockito.never()).cancel(true); } @Test - public void testRenewLeaseOwnershipLostInterruptsActiveStreamAndShutsDownExecutor() { + public void testRenewLeaseOwnershipLostInterruptsActiveStreamAndCancelsHeartbeat() { ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); - Map activeStreams = new ConcurrentHashMap<>(); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + when(mockExecutor.scheduleWithFixedDelay(any(), anyLong(), anyLong(), any())) + .thenReturn(mockFuture); + UploadId uploadId = new UploadId("upload-lost"); + Map activeStreams = new ConcurrentHashMap<>(); - me.desair.tus.server.util.InterruptibleInputStream stream = - new me.desair.tus.server.util.InterruptibleInputStream( - new ByteArrayInputStream("test".getBytes())); - activeStreams.put("/files/upload-lost", stream); + InterruptibleInputStream stream = + new InterruptibleInputStream(new ByteArrayInputStream("test".getBytes())); + activeStreams.put(uploadId, stream); LeaseData leaseData = new LeaseData( "holder-lost", "/files/upload-lost", 10000L, System.currentTimeMillis() + 10000L); TestLeaseLock lock = - new TestLeaseLock(leaseData, activeStreams, mockExecutor, "test-watchdog") { + new TestLeaseLock(leaseData, uploadId, activeStreams, mockExecutor) { @Override protected void doRenewLease() { throw new IllegalStateException("Lease taken over by rival holder"); @@ -222,6 +231,6 @@ protected void doRenewLease() { lock.renewLease(); assertTrue(stream.isInterrupted()); - verify(mockExecutor).shutdownNow(); + verify(mockFuture).cancel(true); } } diff --git a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java index 7b4bbb74..599d5e38 100644 --- a/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/AbstractLeaseLockingServiceTest.java @@ -6,6 +6,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; @@ -31,17 +32,15 @@ private static class TestLeaseLockingService extends AbstractLeaseLockingService private boolean isExpired = false; private boolean evictShouldSucceed = true; private final List stopSignalsWritten = new ArrayList<>(); - private final List stopSignalsChecked = new ArrayList<>(); + private final List stopSignalsChecked = new ArrayList<>(); private final AtomicBoolean cleanedUp = new AtomicBoolean(false); - private final List cleanedUpLocks = new ArrayList<>(); public TestLeaseLockingService() { super(new UuidUploadIdFactory(), 30000L, 1000L, null, "test-lease-locking-service-watchdog"); } - @Override - public void cleanupLock(UploadId id) throws IOException { - cleanedUpLocks.add(id); + public TestLeaseLockingService(java.util.concurrent.ScheduledExecutorService watchdogExecutor) { + super(new UuidUploadIdFactory(), 30000L, 1000L, "test-shutdown-hook", watchdogExecutor); } @Override @@ -68,8 +67,8 @@ protected void writeStopSignal(UploadId uploadId) { } @Override - protected void checkStopSignalForEntry(String uri, InputStream inputStream) { - stopSignalsChecked.add(uri); + protected void checkStopSignalForUpload(UploadId uploadId, InputStream inputStream) { + stopSignalsChecked.add(uploadId); } @Override @@ -224,18 +223,23 @@ public void testRegisterInputStreamAndRequestLockRelease() throws Exception { service.requestLockRelease(uri); assertThat(service.stopSignalsWritten.size(), is(1)); + + service.cleanupLock(null); + service.cleanupLock(new UploadId(UUID.randomUUID().toString())); } @Test public void testCheckStopSignalsIteratesActiveStreams() { String uri1 = "/files/" + UUID.randomUUID(); String uri2 = "/files/" + UUID.randomUUID(); + UploadId id1 = new UuidUploadIdFactory().readUploadId(uri1); + UploadId id2 = new UuidUploadIdFactory().readUploadId(uri2); service.registerInputStream(uri1, new ByteArrayInputStream("1".getBytes())); service.registerInputStream(uri2, new ByteArrayInputStream("2".getBytes())); service.testCheckStopSignals(); - assertTrue(service.stopSignalsChecked.contains(uri1)); - assertTrue(service.stopSignalsChecked.contains(uri2)); + assertTrue(service.stopSignalsChecked.contains(id1)); + assertTrue(service.stopSignalsChecked.contains(id2)); } @Test @@ -305,21 +309,30 @@ public void testCleanupOnClose() throws Exception { service.close(); assertTrue(service.cleanedUp.get()); - assertTrue(service.isClosed()); } @Test - public void testCleanupLockByUriDelegatesToUploadId() throws Exception { - UUID uuid = UUID.randomUUID(); - UploadId id = new UploadId(uuid); - String uri = "/files/" + uuid; - service.cleanupLock(uri); - - assertEquals(1, service.cleanedUpLocks.size()); - assertEquals(id, service.cleanedUpLocks.get(0)); - - service.cleanupLock((String) null); - assertEquals(2, service.cleanedUpLocks.size()); - assertThat(service.cleanedUpLocks.get(1), is(nullValue())); + public void testCustomWatchdogExecutorConstructorAndGetters() throws Exception { + java.util.concurrent.ScheduledExecutorService mockExec = + mock(java.util.concurrent.ScheduledExecutorService.class); + TestLeaseLockingService customExecService = new TestLeaseLockingService(mockExec); + assertEquals(mockExec, customExecService.getWatchdogExecutor()); + customExecService.close(); + } + + @Test + public void testLockUploadById() throws Exception { + org.junit.Assert.assertNull(service.lockUpload(null)); + UploadId id = new UploadId(UUID.randomUUID().toString()); + UploadLock lock = service.lockUpload(id); + assertNotNull(lock); + } + + @Test(expected = UploadAlreadyLockedException.class) + public void testLockUploadByIdThrowsWhenLocked() throws Exception { + service.acquireShouldSucceed = false; + service.isExpired = false; + UploadId id = new UploadId(UUID.randomUUID().toString()); + service.lockUpload(id); } } diff --git a/src/test/java/me/desair/tus/server/upload/NoOpUploadLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/NoOpUploadLockingServiceTest.java index a86f1eb6..87224be2 100644 --- a/src/test/java/me/desair/tus/server/upload/NoOpUploadLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/NoOpUploadLockingServiceTest.java @@ -36,7 +36,6 @@ public void testNoOpLifecycleAndRegistrationMethodsExecuteCleanly() throws Excep // KISS: verifying method executes cleanly without throwing an exception service.cleanupStaleLocks(); - service.cleanupLock(id); service.registerInputStream("/files/123", new ByteArrayInputStream(new byte[0])); service.requestLockRelease("/files/123"); service.setIdFactory(new UuidUploadIdFactory()); diff --git a/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java index c0ecfe92..f30b34ef 100644 --- a/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/UploadLockingServiceTest.java @@ -33,8 +33,6 @@ public void setIdFactory(UploadIdFactory idFactory) {} // Test default methods for coverage dummyService.registerInputStream("/test/upload/123", new ByteArrayInputStream(new byte[0])); dummyService.requestLockRelease("/test/upload/123"); - dummyService.cleanupLock(new UploadId("123")); - dummyService.cleanupLock("/test/upload/123"); dummyService.close(); assertFalse(dummyService.isLocked(new UploadId("123"))); diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java index 875edeaa..5dd9fab0 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobConcatenationServiceTest.java @@ -297,4 +297,27 @@ public void getAuthorizedBlobUrlShouldFallbackToRawUrlWhenSasIsNull() { String result = concatenationService.getAuthorizedBlobUrl(mockBlob); assertEquals("https://account.blob.core.windows.net/container/blob", result); } + + @Test + public void testResolvePartialBlobNameVariants() { + UploadInfo partialWithDup = new UploadInfo(); + partialWithDup.setId(new UploadId("child-id")); + partialWithDup.setDuplicatesUploadId(new UploadId("parent-id")); + + UploadInfo partialNormal = new UploadInfo(); + partialNormal.setId(new UploadId("normal-id")); + + assertEquals("uploads/parent-id", concatenationService.resolvePartialBlobName(partialWithDup)); + assertEquals("uploads/normal-id", concatenationService.resolvePartialBlobName(partialNormal)); + assertEquals("uploads/", concatenationService.resolvePartialBlobName(null)); + + // When storageService is AzureBlobStorageService + AzureBlobStorageService mockAzureStorage = mock(AzureBlobStorageService.class); + when(mockAzureStorage.getAzureBlobName(partialWithDup)).thenReturn("custom-blob-name"); + AzureBlobConcatenationService serviceWithAzureStorage = + new AzureBlobConcatenationService(containerClient, "uploads/", mockAzureStorage); + + assertEquals( + "custom-blob-name", serviceWithAzureStorage.resolvePartialBlobName(partialWithDup)); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java index 2da5badb..35c2d804 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobLockingServiceTest.java @@ -39,7 +39,9 @@ public void setUp() { .endpoint("https://dummyaccount.blob.core.windows.net") .containerName("dummy-container") .buildClient(); - lockingService = new AzureBlobLockingService(containerClient); + lockingService = + new AzureBlobLockingService( + containerClient, AzureBlobLockingService.DEFAULT_LOCKS_PREFIX, null); TimeBasedUploadIdFactory idFactory = new TimeBasedUploadIdFactory(); idFactory.setUploadUri("/test/upload"); lockingService.setIdFactory(idFactory); @@ -51,16 +53,22 @@ public void constructorShouldThrowOnNullContainerClient() { } @Test - public void prefixSanitizationVariants() { + public void prefixSanitizationVariants() throws Exception { AzureBlobLockingService service1 = new AzureBlobLockingService(containerClient, null); AzureBlobLockingService service2 = new AzureBlobLockingService(containerClient, "/custom/locks"); AzureBlobLockingService service3 = new AzureBlobLockingService(containerClient, "custom/locks/"); - assertNotNull(service1); - assertNotNull(service2); - assertNotNull(service3); + try { + assertNotNull(service1); + assertNotNull(service2); + assertNotNull(service3); + } finally { + service1.close(); + service2.close(); + service3.close(); + } } @Test(expected = NullPointerException.class) @@ -97,14 +105,6 @@ public void requestLockReleaseShouldDoNothingOnInvalidUri() { lockingService.requestLockRelease(null); } - @Test - public void cleanupLockWithNullOrInvalidShouldDoNothing() throws Exception { - // KISS: verifying null/invalid parameters are handled safely without throwing exceptions - lockingService.cleanupLock((UploadId) null); - lockingService.cleanupLock((String) null); - lockingService.cleanupLock("invalid-uri-no-id"); - } - @Test public void deleteLockBlobIfUnleasedNullReturnsFalse() { assertFalse(lockingService.deleteLockBlobIfUnleased(null)); @@ -126,43 +126,6 @@ public void deleteLockBlobWithLeaseExceptionHandled() { assertFalse(lockingService.deleteLockBlobWithLease(mockBlob, "lease-id")); } - @Test - public void cleanupLockWithActiveLockDeletesWithLease() throws Exception { - UploadId testId = new UploadId("test-active-lock-id"); - BlobClient mockBlob = Mockito.mock(BlobClient.class); - com.azure.storage.blob.specialized.BlobLeaseClient realLease = - new com.azure.storage.blob.specialized.BlobLeaseClientBuilder() - .blobClient(mockBlob) - .leaseId("mock-lease-123") - .buildClient(); - - AzureBlobUploadLock mockLock = Mockito.mock(AzureBlobUploadLock.class); - Mockito.when(mockLock.getLeaseClient()).thenReturn(realLease); - - AzureBlobLockingService testService = - new AzureBlobLockingService(containerClient) { - @Override - BlobClient getBlobClient(String blobName) { - return mockBlob; - } - - @Override - void deleteStopSignalBlob(String idStr) { - // KISS: no-op in unit test to avoid network call against dummy endpoint - } - - @Override - boolean deleteLockBlobWithLease(BlobClient lockBlob, String leaseId) { - return true; - } - }; - - testService.activeLocks.put("test-active-lock-id", mockLock); - testService.cleanupLock(testId); - - Mockito.verify(mockLock).markBlobDeleted(); - } - @Test public void closeShouldCleanUpResources() throws Exception { // KISS: verifying close executes idempotently without throwing an exception @@ -192,19 +155,6 @@ public void testPollStopSignalsCleansStaleWeakReferences() { assertFalse(lockingService.activeStreams.containsKey("stale-upload-id")); } - @Test - public void cleanupLockHandlesExceptionGracefully() throws Exception { - AzureBlobLockingService throwingService = - new AzureBlobLockingService(containerClient) { - @Override - void deleteStopSignalBlob(String idStr) { - throw new RuntimeException("Simulated delete stop failure"); - } - }; - // KISS: verify method catches exception and logs debug without propagating - throwingService.cleanupLock(new UploadId("12345")); - } - @Test public void deleteLockBlobIfUnleasedCatchesExceptionAndReturnsFalse() { BlobClient mockBlob = Mockito.mock(BlobClient.class); @@ -308,4 +258,175 @@ private BlobItem createTestBlobItemWithNullProps(String name) { new com.azure.storage.blob.implementation.models.BlobItemInternal().setName(blobName); return com.azure.storage.blob.implementation.util.ModelHelper.populateBlobItem(itemInternal); } + + @Test + public void testDeleteStopSignalBlobCatchesException() { + AzureBlobLockingService service = + new AzureBlobLockingService(containerClient) { + @Override + BlobClient getBlobClient(String name) { + BlobClient mock = Mockito.mock(BlobClient.class); + Mockito.doThrow(new RuntimeException("Delete stop error")).when(mock).deleteIfExists(); + return mock; + } + }; + // KISS: verify method swallows exception without throwing + service.deleteStopSignalBlob("upload-test"); + } + + @Test + public void deleteLockBlobIfUnleasedAvailableLeaseStateDeletes() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + Mockito.when(mockBlob.exists()).thenReturn(true); + com.azure.storage.blob.models.BlobProperties props = + createTestBlobProperties(LeaseStateType.AVAILABLE); + Mockito.when(mockBlob.getProperties()).thenReturn(props); + + assertTrue(lockingService.deleteLockBlobIfUnleased(mockBlob)); + Mockito.verify(mockBlob).deleteIfExists(); + } + + @Test + public void deleteLockBlobIfUnleasedLeasedDoesNotDelete() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + Mockito.when(mockBlob.exists()).thenReturn(true); + com.azure.storage.blob.models.BlobProperties props = + createTestBlobProperties(LeaseStateType.LEASED); + Mockito.when(mockBlob.getProperties()).thenReturn(props); + + assertFalse(lockingService.deleteLockBlobIfUnleased(mockBlob)); + Mockito.verify(mockBlob, Mockito.never()).deleteIfExists(); + } + + @Test + public void deleteLockBlobWithLeaseSuccess() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + assertTrue(lockingService.deleteLockBlobWithLease(mockBlob, "valid-lease-id")); + Mockito.verify(mockBlob) + .deleteWithResponse(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void ensureLockBlobExistsCatchesBlobStorageException() { + BlobClient mockBlob = Mockito.mock(BlobClient.class); + com.azure.core.http.HttpResponse response = + Mockito.mock(com.azure.core.http.HttpResponse.class); + Mockito.when(response.getStatusCode()).thenReturn(500); + com.azure.storage.blob.models.BlobStorageException bse = + new com.azure.storage.blob.models.BlobStorageException("Internal error", response, null); + Mockito.doThrow(bse) + .when(mockBlob) + .upload(Mockito.any(com.azure.core.util.BinaryData.class), Mockito.anyBoolean()); + lockingService.ensureLockBlobExists(mockBlob); + } + + private com.azure.storage.blob.models.BlobProperties createTestBlobProperties( + LeaseStateType leaseState) { + return new com.azure.storage.blob.models.BlobProperties( + null, + OffsetDateTime.now(), + "etag", + 0L, + null, + null, + null, + null, + null, + null, + null, + null, + null, + leaseState, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null); + } + + @Test + public void testPollStopSignalsCatchesException() { + ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + AzureBlobLockingService service = + new AzureBlobLockingService( + containerClient, AzureBlobLockingService.DEFAULT_LOCKS_PREFIX, null) { + @Override + BlobClient getBlobClient(String name) { + throw new RuntimeException("Stop blob check failed"); + } + }; + service.activeStreams.put("poll-err", new java.lang.ref.WeakReference<>(stream)); + service.pollStopSignals(); + assertNotNull(service); + } + + @Test + public void testPollStopSignalsDetectsStopBlobAndInterrupts() { + ByteArrayInputStream bais = new ByteArrayInputStream("test".getBytes()); + InterruptibleInputStream stream = new InterruptibleInputStream(bais); + + BlobClient mockStopBlob = Mockito.mock(BlobClient.class); + Mockito.when(mockStopBlob.exists()).thenReturn(true); + + AzureBlobLockingService service = + new AzureBlobLockingService( + containerClient, AzureBlobLockingService.DEFAULT_LOCKS_PREFIX, null) { + @Override + BlobClient getBlobClient(String name) { + return mockStopBlob; + } + }; + service.activeStreams.put("poll-stop", new java.lang.ref.WeakReference<>(stream)); + service.pollStopSignals(); + + assertTrue(stream.isInterrupted()); + Mockito.verify(mockStopBlob).deleteIfExists(); + } + + @Test + public void testLockUploadNullReturnsNull() throws Exception { + assertNull(lockingService.lockUpload(null)); + } + + @Test + public void testCleanupLock() { + BlobClient mockLock = Mockito.mock(BlobClient.class); + BlobClient mockStop = Mockito.mock(BlobClient.class); + AzureBlobLockingService service = + new AzureBlobLockingService(containerClient) { + @Override + BlobClient getBlobClient(String name) { + if (name.endsWith(".stop")) { + return mockStop; + } + return mockLock; + } + }; + + service.cleanupLock(null); + + UploadId id = new UploadId("test-id"); + service.cleanupLock(id); + Mockito.verify(mockStop).deleteIfExists(); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java index b16bfd12..be27bbca 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobStorageServiceTest.java @@ -5,6 +5,11 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobContainerClientBuilder; @@ -15,6 +20,11 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.exception.MaxAppendSizeExceededException; import me.desair.tus.server.upload.TimeBasedUploadIdFactory; @@ -380,4 +390,42 @@ public void testSetUploadLockingServiceNullThrowsNpe() { public void testGetUploadLockingServiceReturnsConfiguredInstance() { assertNotNull(storageService.getUploadLockingService()); } + + @Test + public void testUploadExecutorRejectedExecutionHandler() throws Exception { + ThreadPoolExecutor executor = storageService.getUploadExecutor(); + RejectedExecutionHandler handler = executor.getRejectedExecutionHandler(); + assertNotNull(handler); + + // 1. Rejected when executor is shut down + ThreadPoolExecutor mockExecutor = mock(ThreadPoolExecutor.class); + when(mockExecutor.isShutdown()).thenReturn(true); + try { + handler.rejectedExecution(() -> {}, mockExecutor); + fail("Expected RejectedExecutionException when shut down"); + } catch (RejectedExecutionException e) { + assertTrue(e.getMessage().contains("shut down")); + } + + // 2. Enqueued when executor is active + BlockingQueue queue = new LinkedBlockingQueue<>(); + when(mockExecutor.isShutdown()).thenReturn(false); + when(mockExecutor.getQueue()).thenReturn(queue); + Runnable task = () -> {}; + handler.rejectedExecution(task, mockExecutor); + assertEquals(1, queue.size()); + + // 3. Interrupted while enqueuing + @SuppressWarnings("unchecked") + BlockingQueue fullQueue = mock(BlockingQueue.class); + doThrow(new InterruptedException("interrupted")).when(fullQueue).put(any(Runnable.class)); + when(mockExecutor.getQueue()).thenReturn(fullQueue); + try { + handler.rejectedExecution(task, mockExecutor); + fail("Expected RejectedExecutionException on interrupt"); + } catch (RejectedExecutionException e) { + assertTrue(e.getMessage().contains("Interrupted")); + assertTrue(Thread.interrupted()); // clears interrupted flag + } + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java index 10ef2a7f..ff132cb5 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureBlobUploadLockTest.java @@ -2,7 +2,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import com.azure.core.http.HttpHeaders; @@ -15,6 +19,7 @@ import com.azure.storage.blob.specialized.BlobLeaseClient; import com.azure.storage.blob.specialized.BlobLeaseClientBuilder; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import org.junit.Before; import org.junit.Test; @@ -56,13 +61,19 @@ public void constructorShouldThrowOnNullUploadUri() { @Test public void testLockGettersAndReleaseWithExecutor() throws Exception { ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + doReturn(mockFuture) + .when(mockExecutor) + .scheduleWithFixedDelay(any(), anyLong(), anyLong(), any()); + AzureBlobUploadLock lock = new AzureBlobUploadLock(null, null, "/test/upload/12345", mockExecutor); assertEquals("/test/upload/12345", lock.getUploadUri()); lock.release(); - verify(mockExecutor).shutdownNow(); + verify(mockFuture).cancel(true); + verify(mockExecutor, never()).shutdownNow(); // Secondary release or close should be idempotent lock.release(); @@ -119,6 +130,10 @@ public void testReleaseRemovesActiveStreamRegistration() { @Test public void testRenewLeaseFailureInterruptsActiveStream() { ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + doReturn(mockFuture) + .when(mockExecutor) + .scheduleWithFixedDelay(any(), anyLong(), anyLong(), any()); java.util.Map< String, java.lang.ref.WeakReference> activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); @@ -150,7 +165,8 @@ void executeRenew() { org.junit.Assert.assertTrue(stream.isInterrupted()); org.junit.Assert.assertFalse(activeStreams.containsKey("upload-failing")); - verify(mockExecutor).shutdownNow(); + verify(mockFuture).cancel(true); + verify(mockExecutor, never()).shutdownNow(); } @Test @@ -191,6 +207,10 @@ void executeRenew() { @Test public void testRenewLeaseLostInterruptsActiveStream() { ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + doReturn(mockFuture) + .when(mockExecutor) + .scheduleWithFixedDelay(any(), anyLong(), anyLong(), any()); java.util.Map< String, java.lang.ref.WeakReference> activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); @@ -225,12 +245,17 @@ void executeRenew() { org.junit.Assert.assertTrue(stream.isInterrupted()); org.junit.Assert.assertFalse(activeStreams.containsKey("upload-lost")); - verify(mockExecutor).shutdownNow(); + verify(mockFuture).cancel(true); + verify(mockExecutor, never()).shutdownNow(); } @Test public void testRenewLeaseConflictErrorInterruptsStream() { ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + doReturn(mockFuture) + .when(mockExecutor) + .scheduleWithFixedDelay(any(), anyLong(), anyLong(), any()); java.util.Map< String, java.lang.ref.WeakReference> activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); @@ -267,12 +292,17 @@ void executeRenew() { org.junit.Assert.assertTrue(stream.isInterrupted()); org.junit.Assert.assertFalse(activeStreams.containsKey("upload-conflict")); - verify(mockExecutor).shutdownNow(); + verify(mockFuture).cancel(true); + verify(mockExecutor, never()).shutdownNow(); } @Test public void testRenewLeaseLostWithoutActiveStreamHandlesCleanly() { ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + doReturn(mockFuture) + .when(mockExecutor) + .scheduleWithFixedDelay(any(), anyLong(), anyLong(), any()); // activeStreams and uploadId both null AzureBlobUploadLock lockWithoutStreams = @@ -285,13 +315,18 @@ void executeRenew() { lockWithoutStreams.setLeaseExpiresAt(System.currentTimeMillis() - 1000L); lockWithoutStreams.renewLease(); - verify(mockExecutor).shutdownNow(); + verify(mockFuture).cancel(true); + verify(mockExecutor, never()).shutdownNow(); // activeStreams present but no stream registered for uploadId java.util.Map< String, java.lang.ref.WeakReference> activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); ScheduledExecutorService mockExecutor2 = mock(ScheduledExecutorService.class); + ScheduledFuture mockFuture2 = mock(ScheduledFuture.class); + doReturn(mockFuture2) + .when(mockExecutor2) + .scheduleWithFixedDelay(any(), anyLong(), anyLong(), any()); AzureBlobUploadLock lockWithEmptyStreams = new AzureBlobUploadLock( @@ -304,7 +339,8 @@ void executeRenew() { lockWithEmptyStreams.setLeaseExpiresAt(System.currentTimeMillis() - 1000L); lockWithEmptyStreams.renewLease(); - verify(mockExecutor2).shutdownNow(); + verify(mockFuture2).cancel(true); + verify(mockExecutor2, never()).shutdownNow(); } @Test @@ -338,4 +374,187 @@ public void testReleaseRemovesActiveLockAndSkipsReleaseWhenBlobDeleted() { // Verify activeLocks is cleared org.junit.Assert.assertFalse(activeLocks.containsKey("lock-deleted-test")); } + + @Test + public void testConstructors5ArgAnd6Arg() { + AzureBlobUploadLock lock5 = + new AzureBlobUploadLock( + leaseClient, + lockBlob, + "/test/upload/5", + "5", + new java.util.concurrent.ConcurrentHashMap<>()); + assertNotNull(lock5); + + AzureBlobUploadLock lock6 = + new AzureBlobUploadLock( + leaseClient, + lockBlob, + "/test/upload/6", + "6", + new java.util.concurrent.ConcurrentHashMap<>(), + new java.util.concurrent.ConcurrentHashMap<>()); + assertNotNull(lock6); + } + + @Test + public void testReleaseLockResourceBlobStorageExceptionFallbackToReleaseLease() { + BlobClient mockBlob = mock(BlobClient.class); + BlobClient mockStopBlob = mock(BlobClient.class); + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + + HttpResponse response = mock(HttpResponse.class); + org.mockito.Mockito.when(response.getStatusCode()).thenReturn(500); + BlobStorageException bse = new BlobStorageException("Server error", response, null); + org.mockito.Mockito.doThrow(bse).when(mockBlob).deleteWithResponse(any(), any(), any(), any()); + + boolean[] fallbackCalled = new boolean[] {false}; + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + leaseClient, + mockBlob, + mockStopBlob, + "/test/upload/fallback-test", + mockExecutor, + "fallback-test", + null, + null) { + @Override + void fallbackReleaseLease() { + fallbackCalled[0] = true; + } + }; + + lock.release(); + org.junit.Assert.assertTrue(fallbackCalled[0]); + verify(mockStopBlob).deleteIfExists(); + } + + @Test + public void testReleaseLockResourceGenericExceptionFallbackToReleaseLease() { + BlobClient mockBlob = mock(BlobClient.class); + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + + org.mockito.Mockito.doThrow(new RuntimeException("Network error")) + .when(mockBlob) + .deleteWithResponse(any(), any(), any(), any()); + + boolean[] fallbackCalled = new boolean[] {false}; + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + leaseClient, + mockBlob, + "/test/upload/generic-fallback", + mockExecutor, + "generic-fallback", + null, + null) { + @Override + void fallbackReleaseLease() { + fallbackCalled[0] = true; + } + }; + + lock.release(); + org.junit.Assert.assertTrue(fallbackCalled[0]); + } + + @Test + public void testReleaseLockResourceBlobStorageException404DoesNotFallback() { + BlobClient mockBlob = mock(BlobClient.class); + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + + HttpResponse response = mock(HttpResponse.class); + org.mockito.Mockito.when(response.getStatusCode()).thenReturn(404); + BlobStorageException bse = new BlobStorageException("Not found", response, null); + org.mockito.Mockito.doThrow(bse).when(mockBlob).deleteWithResponse(any(), any(), any(), any()); + + boolean[] fallbackCalled = new boolean[] {false}; + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + leaseClient, mockBlob, "/test/upload/404-test", mockExecutor, "404-test", null, null) { + @Override + void fallbackReleaseLease() { + fallbackCalled[0] = true; + } + }; + + lock.release(); + org.junit.Assert.assertFalse(fallbackCalled[0]); + } + + @Test + public void testRenewLeaseStatus412InterruptsStream() { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + java.util.Map< + String, java.lang.ref.WeakReference> + activeStreams = new java.util.concurrent.ConcurrentHashMap<>(); + + me.desair.tus.server.util.InterruptibleInputStream stream = + new me.desair.tus.server.util.InterruptibleInputStream( + new java.io.ByteArrayInputStream("test".getBytes())); + activeStreams.put("upload-412", new java.lang.ref.WeakReference<>(stream)); + + HttpResponse response = mock(HttpResponse.class); + org.mockito.Mockito.when(response.getStatusCode()).thenReturn(412); + BlobStorageException status412Ex = + new BlobStorageException("Precondition failed", response, null); + + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, null, "/test/upload/upload-412", mockExecutor, "upload-412", activeStreams) { + @Override + void executeRenew() { + throw status412Ex; + } + }; + + lock.setLeaseExpiresAt(System.currentTimeMillis() + 30000L); + lock.renewLease(); + + org.junit.Assert.assertTrue(stream.isInterrupted()); + org.junit.Assert.assertFalse(activeStreams.containsKey("upload-412")); + } + + @Test + public void testFallbackReleaseLeaseWhenLeaseClientNull() { + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, null, "/test/upload/fallback-null", (ScheduledExecutorService) null, null, null); + // KISS: leaseClient is null, verifies method executes cleanly without network calls + lock.fallbackReleaseLease(); + assertNotNull(lock); + } + + @Test + public void testFallbackReleaseLeaseWhenLeaseClientThrows() { + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, + null, + "/test/upload/fallback-throw", + (ScheduledExecutorService) null, + null, + null) { + @Override + void executeReleaseLease() { + throw new RuntimeException("Release failed"); + } + }; + lock.fallbackReleaseLease(); + assertNotNull(lock); + } + + @Test + public void testReleaseHandlesStopBlobExceptionQuietly() { + BlobClient mockStop = mock(BlobClient.class); + org.mockito.Mockito.when(mockStop.deleteIfExists()) + .thenThrow(new RuntimeException("Stop delete error")); + + AzureBlobUploadLock lock = + new AzureBlobUploadLock( + null, null, mockStop, "/test/upload/stop-ex", null, "stop-ex", null, null); + lock.release(); + assertNotNull(lock); + } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java index 6102383d..058293a2 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/azure/AzureUtilsTest.java @@ -1,14 +1,26 @@ package me.desair.tus.server.upload.azure; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.azure.core.http.HttpClient; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobContainerClientBuilder; import com.azure.storage.blob.models.BlobErrorCode; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlockBlobClient; import org.junit.Test; +import reactor.core.publisher.Mono; public class AzureUtilsTest { @@ -23,6 +35,97 @@ private BlobStorageException createException(int statusCode, BlobErrorCode error return new BlobStorageException("Test exception", response, errorCode); } + @Test + public void testSanitizePrefix() { + assertEquals("", AzureUtils.sanitizePrefix(null)); + assertEquals("", AzureUtils.sanitizePrefix("")); + assertEquals("", AzureUtils.sanitizePrefix(" ")); + assertEquals("", AzureUtils.sanitizePrefix("///")); + assertEquals("locks/", AzureUtils.sanitizePrefix("locks")); + assertEquals("locks/", AzureUtils.sanitizePrefix("/locks")); + assertEquals("locks/", AzureUtils.sanitizePrefix("locks/")); + assertEquals("locks/", AzureUtils.sanitizePrefix("/locks/")); + assertEquals("a/b/c/", AzureUtils.sanitizePrefix("/a/b/c")); + assertEquals("custom/", AzureUtils.sanitizePrefix("/custom///")); + } + + @Test + public void testBlobExists() { + // Null checks + assertFalse(AzureUtils.blobExists((BlobClient) null)); + assertFalse(AzureUtils.blobExists((BlobContainerClient) null, "test-blob")); + + HttpClient httpClient = mock(HttpClient.class); + when(httpClient.send(any())).thenReturn(Mono.error(new RuntimeException("Connection error"))); + BlobContainerClient containerClient = + new BlobContainerClientBuilder() + .endpoint("https://dummyaccount.blob.core.windows.net") + .containerName("dummy-container") + .httpClient(httpClient) + .buildClient(); + + assertFalse(AzureUtils.blobExists(containerClient, null)); + + // Blob exists == true + BlobClient blobClient = mock(BlobClient.class); + when(blobClient.exists()).thenReturn(true); + assertTrue(AzureUtils.blobExists(blobClient)); + + // Blob exists == false + when(blobClient.exists()).thenReturn(false); + assertFalse(AzureUtils.blobExists(blobClient)); + + // Blob exists throws exception + when(blobClient.exists()).thenThrow(new RuntimeException("Connection error")); + assertFalse(AzureUtils.blobExists(blobClient)); + + // BlockBlobClient overloads + assertFalse(AzureUtils.blobExists((BlockBlobClient) null)); + BlockBlobClient blockBlobClient = + containerClient.getBlobClient("test-blob").getBlockBlobClient(); + assertFalse(AzureUtils.blobExists(blockBlobClient)); + + // Delegating method with mock HTTP client throwing exception + assertFalse(AzureUtils.blobExists(containerClient, "test-blob")); + } + + @Test + public void testDeleteBlobQuietly() { + // Null checks (should not throw) + AzureUtils.deleteBlobQuietly((BlobClient) null); + AzureUtils.deleteBlobQuietly((BlobContainerClient) null, "test-blob"); + + HttpClient httpClient = mock(HttpClient.class); + when(httpClient.send(any())).thenReturn(Mono.error(new RuntimeException("Connection error"))); + BlobContainerClient containerClient = + new BlobContainerClientBuilder() + .endpoint("https://dummyaccount.blob.core.windows.net") + .containerName("dummy-container") + .httpClient(httpClient) + .buildClient(); + + AzureUtils.deleteBlobQuietly(containerClient, null); + + // Normal deletion on BlobClient mock + BlobClient blobClient = mock(BlobClient.class); + AzureUtils.deleteBlobQuietly(blobClient); + verify(blobClient, times(1)).deleteIfExists(); + + // Exception handled quietly on BlobClient mock + doThrow(new RuntimeException("Azure delete failed")).when(blobClient).deleteIfExists(); + AzureUtils.deleteBlobQuietly(blobClient); + verify(blobClient, times(2)).deleteIfExists(); + + // BlockBlobClient overloads + AzureUtils.deleteBlobQuietly((BlockBlobClient) null); + BlockBlobClient blockBlobClient = + containerClient.getBlobClient("test-blob").getBlockBlobClient(); + AzureUtils.deleteBlobQuietly(blockBlobClient); + + // Delegating method on container client with mock HTTP client throwing exception + AzureUtils.deleteBlobQuietly(containerClient, "test-blob"); + } + @Test public void testParseErrorResponseNull() { assertEquals(AzureErrorType.UNKNOWN, AzureUtils.parseErrorResponse(null)); diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java index fc4c7f36..48f0dd1c 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobLockingService.java @@ -84,6 +84,7 @@ public void testAzureBlobUploadLock3ArgConstructor() throws Exception { com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/" + testId + ".lock"); + lockBlob.upload(com.azure.core.util.BinaryData.fromBytes(new byte[0]), true); com.azure.storage.blob.specialized.BlobLeaseClient lease = new com.azure.storage.blob.specialized.BlobLeaseClientBuilder() .blobClient(lockBlob) @@ -231,98 +232,29 @@ public void testStopSignalBlobCatchBlocksHandledGracefully() { } @Test - public void testCleanupLockDeletesLockAndStopBlobs() throws Exception { - UploadId id = new UploadId(111222L); - UploadLock lock = lockingService.lockUploadByUri("/test/upload/111222"); - assertNotNull(lock); - lockingService.createStopSignalBlob("111222"); - lock.release(); - - com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/111222.lock"); - com.azure.storage.blob.BlobClient stopBlob = containerClient.getBlobClient("locks/111222.stop"); - - assertTrue(Boolean.TRUE.equals(lockBlob.exists())); - assertTrue(Boolean.TRUE.equals(stopBlob.exists())); - - lockingService.cleanupLock(id); - - assertFalse(Boolean.TRUE.equals(lockBlob.exists())); - assertFalse(Boolean.TRUE.equals(stopBlob.exists())); - } - - @Test - public void testCleanupLockByUri() throws Exception { - UploadLock lock = lockingService.lockUploadByUri("/test/upload/333444"); - assertNotNull(lock); - lock.release(); - - com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/333444.lock"); - assertTrue(Boolean.TRUE.equals(lockBlob.exists())); - - lockingService.cleanupLock("/test/upload/333444"); - - assertFalse(Boolean.TRUE.equals(lockBlob.exists())); - } - - @Test - public void testCleanupLockActivelyLeasedBlobByExternalClientIsPreserved() throws Exception { - UploadId id = new UploadId(555666L); - com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/555666.lock"); - lockBlob.upload(new ByteArrayInputStream(new byte[0]), 0); - - // External lease client simulates another replica / pod holding the lease - com.azure.storage.blob.specialized.BlobLeaseClient externalLease = - new com.azure.storage.blob.specialized.BlobLeaseClientBuilder() - .blobClient(lockBlob) - .buildClient(); - externalLease.acquireLease(30); - - assertTrue(Boolean.TRUE.equals(lockBlob.exists())); - - // When actively leased by another node, cleanupLock must NOT delete the lock blob - lockingService.cleanupLock(id); - assertTrue( - "Actively leased blob by another client must not be deleted", - Boolean.TRUE.equals(lockBlob.exists())); - - // After the other client releases the lease, cleanupLock should delete it - externalLease.releaseLease(); - lockingService.cleanupLock(id); - assertFalse("Unleased blob should be deleted", Boolean.TRUE.equals(lockBlob.exists())); - } - - @Test - public void testCleanupLockActivelyLeasedBlobByCurrentServiceIsDeleted() throws Exception { - UploadId id = new UploadId(666777L); + public void testLockReleaseDeletesLockBlobAndStopBlob() throws Exception { UploadLock lock = lockingService.lockUploadByUri("/test/upload/666777"); assertNotNull(lock); com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/666777.lock"); assertTrue(Boolean.TRUE.equals(lockBlob.exists())); - // When actively leased by the current service (e.g. during DELETE termination), - // cleanupLock deletes the blob using the active lease ID - lockingService.cleanupLock(id); - assertFalse( - "Actively leased blob by current service should be deleted", - Boolean.TRUE.equals(lockBlob.exists())); - - // Release after cleanup should execute cleanly without error + // When released, the lock blob is automatically and atomically deleted using the active lease + // ID lock.release(); + assertFalse("Lock blob should be deleted on release", Boolean.TRUE.equals(lockBlob.exists())); } @Test public void testCleanupStaleLocksPreservesFreshBlobs() throws Exception { - UploadLock lock = lockingService.lockUploadByUri("/test/upload/777888"); - assertNotNull(lock); - lock.release(); - com.azure.storage.blob.BlobClient lockBlob = containerClient.getBlobClient("locks/777888.lock"); + lockBlob.upload(com.azure.core.util.BinaryData.fromBytes(new byte[0]), true); assertTrue(Boolean.TRUE.equals(lockBlob.exists())); lockingService.cleanupStaleLocks(); // Fresh blob (created just now, < 120s old) must NOT be deleted by cleanupStaleLocks assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + lockBlob.deleteIfExists(); } } diff --git a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java index 5d9ae13b..9654a523 100644 --- a/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java +++ b/src/test/java/me/desair/tus/server/upload/azure/ITAzureBlobStorageService.java @@ -271,29 +271,32 @@ public void terminateUploadShouldHandleLeasedAndUnleasedLockBlobs() throws Excep storageService.append(created, new ByteArrayInputStream("0123456789".getBytes())); assertNotNull(storageService.getUploadInfo(created.getId())); - // 1. Verify unleased lock blob deletion: create an unleased lock blob and verify it is removed + // 1. Verify unleased lock blob: terminateUpload terminates data and metadata; unleased lock + // blob is cleaned up via lockingService.cleanupLock BlobClient lockBlob = containerClient.getBlobClient("locks/" + created.getId() + ".lock"); lockBlob.upload(BinaryData.fromBytes("lock".getBytes(StandardCharsets.UTF_8)), true); assertTrue(Boolean.TRUE.equals(lockBlob.exists())); storageService.terminateUpload(created); assertNull(storageService.getUploadInfo(created.getId())); + + AzureBlobLockingService lockingService = new AzureBlobLockingService(containerClient); + lockingService.cleanupLock(created.getId()); assertFalse(Boolean.TRUE.equals(lockBlob.exists())); - // 2. Verify leased lock blob: create upload, hold active lease, and verify terminateUpload - // skips deletion so Azure HTTP 412 (LeaseIdMissing) error is never triggered or logged + // 2. Verify leased lock blob: create upload, hold active lease, and verify UploadLock release + // deletes the active lease blob cleanly upon close UploadInfo info2 = new UploadInfo(); info2.setLength(10L); UploadInfo created2 = storageService.create(info2, "owner1"); - AzureBlobLockingService lockingService = new AzureBlobLockingService(containerClient); - UploadLock lock = lockingService.lockUploadByUri("/test/upload/" + created2.getId()); - try { + BlobClient lockBlob2 = containerClient.getBlobClient("locks/" + created2.getId() + ".lock"); + try (UploadLock lock = lockingService.lockUploadByUri("/test/upload/" + created2.getId())) { + assertTrue(Boolean.TRUE.equals(lockBlob2.exists())); storageService.terminateUpload(created2); assertNull(storageService.getUploadInfo(created2.getId())); - } finally { - lock.release(); } + assertFalse(Boolean.TRUE.equals(lockBlob2.exists())); } @Test @@ -390,6 +393,25 @@ public void cleanupExpiredUploadsShouldDeleteExpired() throws Exception { assertFalse(Boolean.TRUE.equals(lockBlob.exists())); } + @Test + public void cleanupExpiredUploadsNoArgShouldDeleteExpiredAndCleanLock() throws Exception { + storageService.setUploadExpirationPeriod(1L); + + UploadInfo info = new UploadInfo(); + info.setLength(20L); // In-progress upload + UploadInfo created = storageService.create(info, "owner1"); + + com.azure.storage.blob.BlobClient lockBlob = + containerClient.getBlobClient("locks/" + created.getId() + ".lock"); + lockBlob.upload(com.azure.core.util.BinaryData.fromString(""), true); + assertTrue(Boolean.TRUE.equals(lockBlob.exists())); + + Thread.sleep(50L); + storageService.cleanupExpiredUploads(); + assertNull(storageService.getUploadInfo(created.getId())); + assertFalse(Boolean.TRUE.equals(lockBlob.exists())); + } + @Test public void cleanupExpiredUploadsShouldPreserveCompletedUploads() throws Exception { storageService.setUploadExpirationPeriod(1L); @@ -680,6 +702,20 @@ public void testCleanupExpiredUploadsSkipsCurrentlyLockedUploads() throws Except UploadLock lock = lockingService.lockUploadByUri("/test/upload/" + expiredLocked.getId()); assertNotNull(lock); + // 3. Create an orphaned corrupted metadata blob with an existing lock to verify cleanupLock on + // null info + String corruptIdStr = "corrupt-azure-" + System.nanoTime(); + containerClient + .getBlobClient("metadata/" + corruptIdStr + ".info") + .upload(BinaryData.fromString("bad-json"), true); + containerClient + .getBlobClient("locks/" + corruptIdStr + ".lock") + .upload(BinaryData.fromString("dummy-lock"), true); + containerClient + .getBlobClient("locks/" + corruptIdStr + ".stop") + .upload(BinaryData.fromString("dummy-stop"), true); + assertTrue(containerClient.getBlobClient("locks/" + corruptIdStr + ".lock").exists()); + try { storageService.cleanupExpiredUploads(lockingService); @@ -688,6 +724,10 @@ public void testCleanupExpiredUploadsSkipsCurrentlyLockedUploads() throws Except // Unlocked upload SHOULD have been cleaned up assertNull(storageService.getUploadInfo(expiredUnlocked.getId())); + + // Orphaned lock and stop signal for corrupted upload SHOULD have been cleaned up + assertFalse(containerClient.getBlobClient("locks/" + corruptIdStr + ".lock").exists()); + assertFalse(containerClient.getBlobClient("locks/" + corruptIdStr + ".stop").exists()); } finally { lock.release(); } @@ -890,6 +930,11 @@ public void testTerminateUploadWithLocksAndStopBlob() throws Exception { lockBlob.upload(BinaryData.fromString("lock"), true); storageService.terminateUpload(created); + assertNull(storageService.getUploadInfo(created.getId())); + + // Orphaned locks and stop blobs are cleaned up via locking service cleanup + AzureBlobLockingService lockingService = new AzureBlobLockingService(containerClient); + lockingService.cleanupLock(created.getId()); assertFalse(stopBlob.exists()); assertFalse(lockBlob.exists()); @@ -1019,6 +1064,11 @@ public void testTerminateUploadWithUnleasedLockBlob() throws Exception { assertTrue(Boolean.TRUE.equals(lockBlob.exists())); storageService.terminateUpload(created); + assertNull(storageService.getUploadInfo(created.getId())); + + // Orphaned unleased lock blob is cleaned up via locking service cleanup + AzureBlobLockingService lockingService = new AzureBlobLockingService(containerClient); + lockingService.cleanupLock(created.getId()); assertFalse(Boolean.TRUE.equals(lockBlob.exists())); } diff --git a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java index a1e7edf1..bbb3ff8c 100644 --- a/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/cache/ThreadLocalCachedStorageAndLockingServiceTest.java @@ -255,12 +255,6 @@ public void testDelegateMethods() throws Exception { service.requestLockRelease("/files/1"); verify(mockLocking, times(1)).requestLockRelease("/files/1"); - service.cleanupLock(id); - verify(mockLocking, times(1)).cleanupLock(id); - - service.cleanupLock("/files/1"); - verify(mockLocking, times(1)).cleanupLock("/files/1"); - service.getUploadLockingService(); verify(mockStorage, times(1)).getUploadLockingService(); @@ -279,4 +273,11 @@ public void testDelegateMethods() throws Exception { service.close(); verify(mockLocking, times(1)).close(); } + + @Test + public void testCleanupLockDelegates() throws IOException { + UploadId id = new UploadId(UUID.randomUUID().toString()); + service.cleanupLock(id); + verify(mockLocking, times(1)).cleanupLock(id); + } } diff --git a/src/test/java/me/desair/tus/server/upload/checksum/DeduplicationServiceTest.java b/src/test/java/me/desair/tus/server/upload/checksum/DeduplicationServiceTest.java index ba4618f3..63a3f6ba 100644 --- a/src/test/java/me/desair/tus/server/upload/checksum/DeduplicationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/checksum/DeduplicationServiceTest.java @@ -700,10 +700,9 @@ public void testRemoveUploadFromIndexDoesNotInvokeLockCleanup() throws Exception adapter.put(parentId, parentInfo, true); index.tryRegisterParent(checksumKey, parentId); - // Parent terminates with no children; index is removed but lockingService.cleanupLock is NOT - // invoked (delegated to storage service terminateUpload) + // Parent terminates with no children; index is removed cleanly assertTrue(deduplicationService.removeUploadFromIndex(parentInfo)); - org.mockito.Mockito.verify(lockingService, org.mockito.Mockito.never()).cleanupLock(parentId); + assertNull(index.findParent(checksumKey)); } @Test @@ -826,6 +825,68 @@ public void testThreeParamConstructorWithValidLockingService() { assertNotNull(service); } + @Test + public void testLockParentWithRetryInterruptedThrowsTusException() throws Exception { + UploadLockingService mockLocking = mock(UploadLockingService.class); + when(mockLocking.lockUpload(any(UploadId.class))) + .thenThrow(new UploadAlreadyLockedException("locked")); + + DeduplicationService service = new DeduplicationService(index, adapter, mockLocking); + UploadInfo childInfo = new UploadInfo(); + childInfo.setId(new UploadId("child-1")); + childInfo.setDuplicatesUploadId(new UploadId("parent-1")); + + Thread.currentThread().interrupt(); + service.syncParentExpirationIfChild(childInfo); + assertTrue(Thread.interrupted()); // verifies thread interrupted flag was re-asserted + } + + @Test + public void testSyncParentExpirationIfChildNullOrNoParent() { + deduplicationService.syncParentExpirationIfChild(null); + UploadInfo info = new UploadInfo(); + deduplicationService.syncParentExpirationIfChild(info); + assertNull(info.getDuplicatesUploadId()); + } + + @Test + public void testPruneOrphanedIndicesHandlesNullParentAndLockedParent() throws Exception { + ChecksumKey keyWithNullParent = + ChecksumKey.of("1111111111111111111111111111111111111111", ChecksumAlgorithm.SHA1); + index.tryRegisterParent(keyWithNullParent, null); + + ChecksumKey lockedKey = + ChecksumKey.of("2222222222222222222222222222222222222222", ChecksumAlgorithm.SHA1); + UploadId lockedParent = new UploadId("locked-parent"); + index.tryRegisterParent(lockedKey, lockedParent); + + UploadLockingService mockLocking = mock(UploadLockingService.class); + when(mockLocking.lockUpload(lockedParent)) + .thenThrow(new UploadAlreadyLockedException("Parent locked")); + + DeduplicationService service = new DeduplicationService(index, adapter, mockLocking); + // Should skip null parent and skip locked parent cleanly without throwing + service.pruneOrphanedChecksumIndices(); + } + + @Test + public void testPruneOrphanedIndicesRepairsMissingTombstoneWithActiveChildren() throws Exception { + ChecksumKey key = + ChecksumKey.of("3333333333333333333333333333333333333333", ChecksumAlgorithm.SHA1); + UploadId parentId = new UploadId("repair-parent"); + index.tryRegisterParent(key, parentId); + index.addChild(key, new UploadId("child-1")); + + // Parent data exists, but parent info is null, not tombstoned, has children + adapter.put(parentId, null, true); + + deduplicationService.pruneOrphanedChecksumIndices(); + + // Verify tombstone was repaired + assertTrue(index.isParentDeleted(key)); + assertEquals(parentId, index.findParent(key)); + } + private UploadInfo createUploadInfo(UploadId id, ChecksumKey key) { UploadInfo info = new UploadInfo(); info.setId(id); diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java index 9335348f..eb872939 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskLockingServiceTest.java @@ -410,7 +410,7 @@ public void testRequestLockReleaseWithGCedStream() throws Exception { map = (java.util.concurrent.ConcurrentHashMap< String, java.lang.ref.WeakReference>) - field.get(null); + field.get(lockingService); java.lang.ref.WeakReference ref = map.get(uploadIdStr); ref.clear(); @@ -444,7 +444,7 @@ public void testWatchdogRemovesClearedWeakReference() throws Exception { map = (java.util.concurrent.ConcurrentHashMap< String, java.lang.ref.WeakReference>) - field.get(null); + field.get(lockingService); java.lang.ref.WeakReference ref = map.get(uploadIdStr); org.junit.Assert.assertNotNull(ref); @@ -483,7 +483,7 @@ public void testWatchdogInterrupted() throws Exception { java.lang.reflect.Field field = DiskLockingService.class.getDeclaredField("activeLocks"); field.setAccessible(true); java.util.concurrent.ConcurrentHashMap map = - (java.util.concurrent.ConcurrentHashMap) field.get(null); + (java.util.concurrent.ConcurrentHashMap) field.get(lockingService); map.clear(); } @@ -496,7 +496,7 @@ public void testWatchdogUnexpectedException() throws Exception { map = (java.util.concurrent.ConcurrentHashMap< String, java.lang.ref.WeakReference>) - field.get(null); + field.get(lockingService); String uploadIdStr = UUID.randomUUID().toString(); String uri = "/upload/test/" + uploadIdStr; @@ -700,46 +700,7 @@ public void testCloseInterruptsActiveStreams() throws Exception { } @Test - public void testCleanupLockNullAndInvalid() throws Exception { - lockingService.cleanupLock((UploadId) null); - lockingService.cleanupLock((String) null); - // KISS: verifying method executes cleanly without throwing an exception - assertTrue(true); - } - - @Test - public void testCleanupLockDeletesLockAndStopFiles() throws Exception { - UploadId id = new UploadId(UUID.randomUUID().toString()); - Path lockPath = storagePath.resolve("locks").resolve(id.toString()); - Path stopPath = storagePath.resolve("locks").resolve(id.toString() + ".stop"); - Files.createFile(lockPath); - Files.createFile(stopPath); - - assertTrue(Files.exists(lockPath)); - assertTrue(Files.exists(stopPath)); - - lockingService.cleanupLock(id); - - assertFalse(Files.exists(lockPath)); - assertFalse(Files.exists(stopPath)); - } - - @Test - public void testCleanupLockByUri() throws Exception { - UploadId id = new UploadId(UUID.randomUUID().toString()); - when(idFactory.readUploadId(UPLOAD_URL + "/" + id)).thenReturn(id); - - Path lockPath = storagePath.resolve("locks").resolve(id.toString()); - Files.createFile(lockPath); - assertTrue(Files.exists(lockPath)); - - lockingService.cleanupLock(UPLOAD_URL + "/" + id); - - assertFalse(Files.exists(lockPath)); - } - - @Test - public void testCleanupLockWhenActivelyLockedDoesNotDeleteLockFile() throws Exception { + public void testLockReleaseDeletesLockFile() throws Exception { UploadId id = new UploadId(UUID.randomUUID().toString()); when(idFactory.readUploadId(UPLOAD_URL + "/" + id)).thenReturn(id); @@ -749,10 +710,6 @@ public void testCleanupLockWhenActivelyLockedDoesNotDeleteLockFile() throws Exce Path lockPath = storagePath.resolve("locks").resolve(id.toString()); assertTrue(Files.exists(lockPath)); - // When actively locked, cleanupLock must preserve the lock file - lockingService.cleanupLock(id); - assertTrue(Files.exists(lockPath)); - lock.release(); assertFalse(Files.exists(lockPath)); } diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java index cc899fba..e38237a1 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskStorageServiceTest.java @@ -18,6 +18,8 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; @@ -542,7 +544,7 @@ public void cleanupExpiredUploads() throws Exception { assertFalse(Files.exists(getUploadInfoPath(info.getId()))); assertFalse(Files.exists(getStoragePath(info.getId()))); - org.mockito.Mockito.verify(uploadLockingService).cleanupLock(info.getId()); + verify(uploadLockingService).cleanupLock(info.getId()); } @Test @@ -564,7 +566,58 @@ public void cleanupExpiredUploadsNoArg() throws Exception { assertFalse(Files.exists(getUploadInfoPath(info.getId()))); assertFalse(Files.exists(getStoragePath(info.getId()))); - org.mockito.Mockito.verify(uploadLockingService).cleanupLock(info.getId()); + verify(uploadLockingService).cleanupLock(info.getId()); + } + + @Test + public void testCleanupExpiredUploadsHandlesUploadNotFoundException() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(100L); + info.updateExpiration(100L); + + info = storageService.create(info, null); + UploadId uploadId = info.getId(); + assertTrue(Files.exists(getUploadInfoPath(uploadId))); + + when(uploadLockingService.isLocked(any(UploadId.class))).thenReturn(false); + storageService.setUploadLockingService(uploadLockingService); + + Utils.sleep(500L); + + DiskStorageService spyService = spy(storageService); + doThrow(new UploadNotFoundException("Concurrent termination")) + .when(spyService) + .terminateUpload(any(UploadInfo.class)); + + spyService.cleanupExpiredUploads(); + + assertFalse(Files.exists(getStoragePath(uploadId))); + verify(uploadLockingService).cleanupLock(uploadId); + } + + @Test + public void testCleanupExpiredUploadsHandlesNullInfoDuringIteration() throws Exception { + UploadInfo info = new UploadInfo(); + info.setLength(100L); + info.updateExpiration(100L); + + info = storageService.create(info, null); + UploadId uploadId = info.getId(); + assertTrue(Files.exists(getUploadInfoPath(uploadId))); + + when(uploadLockingService.isLocked(any(UploadId.class))).thenReturn(false); + storageService.setUploadLockingService(uploadLockingService); + + Utils.sleep(500L); + + DiskStorageService spyService = spy(storageService); + // ExpiredUploadFilter accepts upload, but subsequent getUploadInfo returns null inside loop + when(spyService.getUploadInfo(uploadId)).thenReturn(info).thenReturn(null); + + spyService.cleanupExpiredUploads(); + + assertFalse(Files.exists(getStoragePath(uploadId))); + verify(uploadLockingService).cleanupLock(uploadId); } @Test diff --git a/src/test/java/me/desair/tus/server/upload/disk/DiskUtilsTest.java b/src/test/java/me/desair/tus/server/upload/disk/DiskUtilsTest.java new file mode 100644 index 00000000..729c5c2a --- /dev/null +++ b/src/test/java/me/desair/tus/server/upload/disk/DiskUtilsTest.java @@ -0,0 +1,347 @@ +package me.desair.tus.server.upload.disk; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.nio.file.spi.FileSystemProvider; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +public class DiskUtilsTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private Path storagePath; + + @Before + public void setUp() throws Exception { + storagePath = temporaryFolder.newFolder("disk-utils-test").toPath(); + } + + @Test + public void testResolveSafePath() { + assertNull(DiskUtils.resolveSafePath(null, "child.txt", "error")); + assertNull(DiskUtils.resolveSafePath(storagePath, null, "error")); + + Path resolved = DiskUtils.resolveSafePath(storagePath, "child.txt", "error"); + assertNotNull(resolved); + assertEquals(storagePath.resolve("child.txt").normalize(), resolved.normalize()); + + // Path traversal attempt directly with .. + try { + DiskUtils.resolveSafePath(storagePath, "../evil.txt", "Path traversal detected"); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertEquals("Path traversal detected", e.getMessage()); + } + + // Path traversal nested + try { + DiskUtils.resolveSafePath(storagePath, "sub/../../evil.txt", "Path traversal detected"); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertEquals("Path traversal detected", e.getMessage()); + } + } + + @Test + public void testInitStorageDirectory() throws Exception { + try { + DiskUtils.initStorageDirectory(null); + fail("Expected NullPointerException on null storagePath"); + } catch (NullPointerException e) { + // Expected + } + + Path newDir = storagePath.resolve("new-storage-dir"); + assertFalse(Files.exists(newDir)); + DiskUtils.initStorageDirectory(newDir); + assertTrue(Files.exists(newDir)); + + // Calling again on existing directory should succeed cleanly + DiskUtils.initStorageDirectory(newDir); + assertTrue(Files.exists(newDir)); + + // When an existing regular file blocks directory creation + Path blockerFile = storagePath.resolve("blocker-file"); + Files.write(blockerFile, "content".getBytes()); + try { + DiskUtils.initStorageDirectory(blockerFile.resolve("nested")); + fail("Expected StoragePathNotAvailableException"); + } catch (StoragePathNotAvailableException e) { + assertTrue(e.getMessage().contains("Unable to create the directory")); + } + } + + @Test + public void testIsSafePathComponent() { + assertFalse(DiskUtils.isSafePathComponent(null)); + assertFalse(DiskUtils.isSafePathComponent("")); + assertFalse(DiskUtils.isSafePathComponent(" ")); + assertFalse(DiskUtils.isSafePathComponent("..")); + assertFalse(DiskUtils.isSafePathComponent("a/b")); + assertFalse(DiskUtils.isSafePathComponent("a\\b")); + assertFalse(DiskUtils.isSafePathComponent("a:b")); + assertFalse(DiskUtils.isSafePathComponent("a*b")); + assertFalse(DiskUtils.isSafePathComponent("a?b")); + assertFalse(DiskUtils.isSafePathComponent("a\"b")); + assertFalse(DiskUtils.isSafePathComponent("ab")); + assertFalse(DiskUtils.isSafePathComponent("a|b")); + assertFalse(DiskUtils.isSafePathComponent("a\u0001b")); + + assertTrue(DiskUtils.isSafePathComponent("valid-file_name.123")); + assertTrue(DiskUtils.isSafePathComponent("uploadId")); + } + + @Test + public void testNormalizeChecksum() { + assertNull(DiskUtils.normalizeChecksum(null)); + + // Hex string should remain unchanged + String hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + assertEquals(hex, DiskUtils.normalizeChecksum(hex)); + + // Base64 encoded SHA-256 (44 chars ending in =) + String base64 = "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; + String expectedHex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + assertEquals(expectedHex, DiskUtils.normalizeChecksum(base64)); + + // Empty string + assertEquals("", DiskUtils.normalizeChecksum("")); + } + + @Test + public void testIsPathOlderThan() throws Exception { + assertFalse(DiskUtils.isPathOlderThan(null, 1000L)); + assertFalse(DiskUtils.isPathOlderThan(null, 1000L, 5000L)); + + Path testFile = storagePath.resolve("age-test.tmp"); + Files.write(testFile, "data".getBytes()); + + long now = System.currentTimeMillis(); + Files.setLastModifiedTime(testFile, FileTime.fromMillis(now - 5000L)); + + assertTrue(DiskUtils.isPathOlderThan(testFile, 4000L, now)); + assertFalse(DiskUtils.isPathOlderThan(testFile, 6000L, now)); + + assertTrue(DiskUtils.isPathOlderThan(testFile, 4000L)); + assertFalse(DiskUtils.isPathOlderThan(testFile, 100000L)); + + Path nonExistent = storagePath.resolve("non-existent-" + UUID.randomUUID()); + assertFalse(DiskUtils.isPathOlderThan(nonExistent, 1000L)); + } + + @Test + public void testDeleteDirectoryQuietly() throws Exception { + assertFalse(DiskUtils.deleteDirectoryQuietly(null)); + Path nonExistent = storagePath.resolve("non-existent-" + UUID.randomUUID()); + assertFalse(DiskUtils.deleteDirectoryQuietly(nonExistent)); + + Path testDir = storagePath.resolve("delete-dir-test-" + UUID.randomUUID()); + Path subDir = testDir.resolve("sub"); + Files.createDirectories(subDir); + Files.write(subDir.resolve("test.txt"), "hello".getBytes()); + + assertTrue(DiskUtils.deleteDirectoryQuietly(testDir)); + assertFalse(Files.exists(testDir)); + } + + @Test + public void testDeleteDirectoryQuietlyIOException() throws Exception { + FileSystemProvider mockProvider = mock(FileSystemProvider.class, Mockito.CALLS_REAL_METHODS); + FileSystem mockFs = mock(FileSystem.class); + when(mockFs.provider()).thenReturn(mockProvider); + + Path mockPath = mock(Path.class); + when(mockPath.getFileSystem()).thenReturn(mockFs); + + Mockito.doNothing().when(mockProvider).checkAccess(mockPath); + + when(mockProvider.readAttributes( + ArgumentMatchers.eq(mockPath), + ArgumentMatchers.eq(BasicFileAttributes.class), + ArgumentMatchers.any())) + .thenThrow(new IOException("Simulated I/O failure")); + + assertFalse(DiskUtils.deleteDirectoryQuietly(mockPath)); + } + + @Test + public void testCreateTempSiblingPath() { + assertThat(DiskUtils.createTempSiblingPath(null), is(nullValue())); + + Path absPath = Paths.get("target", "test-dir", "data.json").toAbsolutePath(); + Path tempAbs = DiskUtils.createTempSiblingPath(absPath); + assertThat(tempAbs, is(notNullValue())); + assertThat(tempAbs.getParent(), is(absPath.getParent())); + assertThat(tempAbs.getFileName().toString().startsWith("data.json.tmp."), is(true)); + + Path relPathNoParent = Paths.get("data.json"); + Path tempRelNoParent = DiskUtils.createTempSiblingPath(relPathNoParent); + assertThat(tempRelNoParent, is(notNullValue())); + assertThat(tempRelNoParent.getFileName().toString().startsWith("data.json.tmp."), is(true)); + } + + @Test + public void testTempPathAutoCloseable() throws Exception { + Path targetFile = Paths.get("target", "temp-target-" + UUID.randomUUID() + ".json"); + Path tempFilePath; + + try (DiskUtils.TempPath tempPath = DiskUtils.createTempSibling(targetFile)) { + tempFilePath = tempPath.getPath(); + assertThat(tempFilePath, is(notNullValue())); + Files.write(tempFilePath, "temp data".getBytes()); + assertThat(Files.exists(tempFilePath), is(true)); + } + + // AutoCloseable should have automatically deleted the temp file + assertThat(Files.exists(tempFilePath), is(false)); + + // Null path handling + try (DiskUtils.TempPath nullTempPath = new DiskUtils.TempPath(null)) { + assertThat(nullTempPath.getPath(), is(nullValue())); + } + } + + @Test + public void testAtomicMove() throws Exception { + // Null inputs should not throw exception + DiskUtils.atomicMove(null, Paths.get("dest")); + DiskUtils.atomicMove(Paths.get("src"), null); + + Path src = Files.createTempFile("tus-disk-atomic-src", ".tmp"); + Path dst = Files.createTempFile("tus-disk-atomic-dst", ".tmp"); + try { + Files.write(src, "atomic test content".getBytes()); + DiskUtils.atomicMove(src, dst); + + assertThat(Files.exists(src), is(false)); + assertThat(Files.exists(dst), is(true)); + assertThat(new String(Files.readAllBytes(dst)), is("atomic test content")); + } finally { + Files.deleteIfExists(src); + Files.deleteIfExists(dst); + } + } + + @Test + public void testReadAndWriteSerializable() throws Exception { + // Null handling + assertThat(DiskUtils.readSerializable(null, TestSerializable.class), is(nullValue())); + DiskUtils.writeSerializable(null, storagePath.resolve("ignored.bin")); + DiskUtils.writeSerializable(new TestSerializable("test"), null); + + // Normal serialization & deserialization + TestSerializable original = new TestSerializable("test-value-123"); + Path file = storagePath.resolve("serializable.bin"); + DiskUtils.writeSerializable(original, file); + + TestSerializable result = DiskUtils.readSerializable(file, TestSerializable.class); + assertThat(result, is(notNullValue())); + assertThat(result.getValue(), is("test-value-123")); + + // Corrupted file + Path corrupted = storagePath.resolve("corrupted.bin"); + Files.write(corrupted, "not a valid serialized object".getBytes()); + assertThat(DiskUtils.readSerializable(corrupted, TestSerializable.class), is(nullValue())); + + // Truncated file + Path truncated = storagePath.resolve("truncated.bin"); + Files.write(truncated, new byte[] {(byte) 0xAC, (byte) 0xED, 0x00}); + assertThat(DiskUtils.readSerializable(truncated, TestSerializable.class), is(nullValue())); + + // Empty file + Path empty = storagePath.resolve("empty.bin"); + Files.createFile(empty); + assertThat(DiskUtils.readSerializable(empty, TestSerializable.class), is(nullValue())); + } + + @Test + public void testLockFileExclusivelyAndShared() throws Exception { + Path tempFile = Files.createTempFile("tus-disk-lock", ".tmp"); + try (FileChannel channel = + FileChannel.open( + tempFile, + java.nio.file.StandardOpenOption.READ, + java.nio.file.StandardOpenOption.WRITE)) { + FileLock lock1 = DiskUtils.lockFileExclusively(channel); + assertThat(lock1, is(notNullValue())); + lock1.release(); + + FileLock lock2 = DiskUtils.lockFileShared(channel); + assertThat(lock2, is(notNullValue())); + lock2.release(); + } finally { + Files.deleteIfExists(tempFile); + } + } + + @Test + public void testInterruptThread() throws Exception { + // Null should handle cleanly + DiskUtils.interruptThread(null); + + // Alive thread + AtomicBoolean wasInterrupted = new AtomicBoolean(false); + Thread thread = + new Thread( + () -> { + try { + Thread.sleep(5000L); + } catch (InterruptedException e) { + wasInterrupted.set(true); + } + }, + "test-disk-thread"); + thread.start(); + Thread.sleep(50L); + + DiskUtils.interruptThread(thread); + thread.join(2000L); + assertTrue(wasInterrupted.get()); + + // Unstarted thread + Thread unstartedThread = new Thread(() -> {}, "test-disk-unstarted"); + DiskUtils.interruptThread(unstartedThread); + } + + public static class TestSerializable implements Serializable { + private static final long serialVersionUID = 1L; + private final String value; + + public TestSerializable(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + } +} diff --git a/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java index 27d0c822..8c0adb09 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/LeaseFileLockingServiceTest.java @@ -170,8 +170,8 @@ public void testExpiredLeaseEvictedAndReacquired() throws Exception { lock.close(); } - @Test(expected = UploadAlreadyLockedException.class) - public void testEmptyLockDirectoryWithinGracePeriodThrowsLocked() throws Exception { + @Test + public void testEmptyLockDirectoryAcquiresImmediatelyWithoutGracePeriodDelay() throws Exception { String uploadIdStr = UUID.randomUUID().toString(); String uri = UPLOAD_URL + "/" + uploadIdStr; @@ -180,8 +180,34 @@ public void testEmptyLockDirectoryWithinGracePeriodThrowsLocked() throws Excepti // Set directory mtime to now (within 5s grace period) Files.setLastModifiedTime(lockDir, FileTime.fromMillis(System.currentTimeMillis())); - // Must be treated as actively being written by another node - lockingService.lockUploadByUri(uri); + // Under exclusive sibling mutex, empty directory is acquired immediately without waiting 5 + // seconds + UploadLock lock = lockingService.lockUploadByUri(uri); + assertNotNull(lock); + lock.close(); + } + + @Test + public void testEmptyLockDirectoryWhenSiblingMutexExistsWithinGracePeriodIsUnexpired() + throws Exception { + String uploadIdStr = UUID.randomUUID().toString(); + Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); + Files.createDirectories(lockDir); + Path mutexDir = storagePath.resolve("locks").resolve(uploadIdStr + ".mutex"); + Files.createDirectories(mutexDir); + Files.setLastModifiedTime(mutexDir, FileTime.fromMillis(System.currentTimeMillis())); + + long now = System.currentTimeMillis(); + // With active sibling mutex within grace period, empty directory is treated as mid-creation + assertFalse(lockingService.isLockDirectoryExpired(lockDir, now)); + + // With active sibling mutex older than grace period, empty directory is expired + Files.setLastModifiedTime(mutexDir, FileTime.fromMillis(now - 10000L)); + assertTrue(lockingService.isLockDirectoryExpired(lockDir, now)); + + DiskUtils.deleteDirectoryQuietly(mutexDir); + // With sibling mutex gone, empty directory is immediately expired + assertTrue(lockingService.isLockDirectoryExpired(lockDir, now)); } @Test @@ -582,7 +608,7 @@ public void testCleanupOnCloseInterruptsActiveStreams() throws Exception { LeaseFileLockingService service = new LeaseFileLockingService(storagePath.toString()); InterruptibleInputStream stream = new InterruptibleInputStream(new ByteArrayInputStream("data".getBytes())); - service.registerInputStream(UPLOAD_URL + "/close-test", stream); + service.registerInputStream(UPLOAD_URL + "/" + UUID.randomUUID(), stream); service.close(); @@ -921,71 +947,75 @@ private LeaseData createExpiredLease( } @Test - public void testCleanupLockNullAndInvalid() throws Exception { - lockingService.cleanupLock((UploadId) null); - lockingService.cleanupLock((String) null); - // KISS: verifying method executes cleanly without throwing an exception - assertTrue(true); - } - - @Test - public void testCleanupLockDeletesExpiredLockDirAndStopFile() throws Exception { + public void testLockReleaseDeletesLockDirectory() throws Exception { String uploadIdStr = UUID.randomUUID().toString(); - UploadId id = new UploadId(uploadIdStr); - Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); - Path stopFile = storagePath.resolve("locks").resolve(uploadIdStr + ".stop"); - Files.createDirectories(lockDir); - Files.write(stopFile, new byte[0]); - - long now = System.currentTimeMillis(); - LeaseData expiredLease = - createExpiredLease( - "old-node", UPLOAD_URL + "/" + uploadIdStr, lockDir.toString(), now - 5000L); - LeaseDataJsonSerializer.serializeToPath(expiredLease, lockDir.resolve("lease.json")); - - assertTrue(Files.exists(lockDir)); - assertTrue(Files.exists(stopFile)); - - lockingService.cleanupLock(id); + String uri = UPLOAD_URL + "/" + uploadIdStr; - assertFalse(Files.exists(lockDir)); - assertFalse(Files.exists(stopFile)); - } + UploadLock lock = lockingService.lockUploadByUri(uri); + assertNotNull(lock); - @Test - public void testCleanupLockByUri() throws Exception { - String uploadIdStr = UUID.randomUUID().toString(); - String uri = UPLOAD_URL + "/" + uploadIdStr; Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); - Files.createDirectories(lockDir); - - long now = System.currentTimeMillis(); - LeaseData expiredLease = createExpiredLease("old-node", uri, lockDir.toString(), now - 5000L); - LeaseDataJsonSerializer.serializeToPath(expiredLease, lockDir.resolve("lease.json")); assertTrue(Files.exists(lockDir)); - lockingService.cleanupLock(uri); - + lock.release(); assertFalse(Files.exists(lockDir)); } @Test - public void testCleanupLockPreservesActivelyHeldLease() throws Exception { - String uploadIdStr = UUID.randomUUID().toString(); - String uri = UPLOAD_URL + "/" + uploadIdStr; - UploadId id = new UploadId(uploadIdStr); - - UploadLock lock = lockingService.lockUploadByUri(uri); - assertNotNull(lock); - - Path lockDir = storagePath.resolve("locks").resolve(uploadIdStr + ".lock"); - assertTrue(Files.exists(lockDir)); + public void testCleanupLockClearsStopSignal() throws Exception { + UploadId id = new UploadId(UUID.randomUUID().toString()); + Path stopFile = storagePath.resolve("locks").resolve(id + ".stop"); + Utils.ensureDirectoryExists(stopFile.getParent()); + Files.write(stopFile, new byte[0]); + assertTrue(Files.exists(stopFile)); - // Actively held lock must not be deleted by cleanupLock + lockingService.cleanupLock(null); lockingService.cleanupLock(id); - assertTrue(Files.exists(lockDir)); + assertFalse(Files.exists(stopFile)); + } - lock.release(); - assertFalse(Files.exists(lockDir)); + @Test + public void testIsLockDirectoryExpiredMutexMtimeIOException() throws Exception { + java.nio.file.spi.FileSystemProvider mockProvider = + org.mockito.Mockito.mock( + java.nio.file.spi.FileSystemProvider.class, org.mockito.Mockito.CALLS_REAL_METHODS); + java.nio.file.FileSystem mockFs = org.mockito.Mockito.mock(java.nio.file.FileSystem.class); + when(mockFs.provider()).thenReturn(mockProvider); + + Path mockLockDir = org.mockito.Mockito.mock(Path.class); + when(mockLockDir.getFileSystem()).thenReturn(mockFs); + + Path mockLeaseFile = org.mockito.Mockito.mock(Path.class); + when(mockLeaseFile.getFileSystem()).thenReturn(mockFs); + when(mockLockDir.resolve("lease.json")).thenReturn(mockLeaseFile); + + Path mockFileName = Paths.get("test-upload.lock"); + when(mockLockDir.getFileName()).thenReturn(mockFileName); + + Path mockMutexDir = org.mockito.Mockito.mock(Path.class); + when(mockMutexDir.getFileSystem()).thenReturn(mockFs); + when(mockLockDir.resolveSibling(org.mockito.ArgumentMatchers.any(String.class))) + .thenReturn(mockMutexDir); + when(mockLockDir.resolveSibling(org.mockito.ArgumentMatchers.any(Path.class))) + .thenReturn(mockMutexDir); + + // lockDir exists + org.mockito.Mockito.doNothing().when(mockProvider).checkAccess(mockLockDir); + // leaseFile does not exist + org.mockito.Mockito.doThrow(new java.nio.file.NoSuchFileException("lease.json")) + .when(mockProvider) + .checkAccess(mockLeaseFile); + // mutexDir exists + org.mockito.Mockito.doNothing().when(mockProvider).checkAccess(mockMutexDir); + + // getLastModifiedTime throws IOException on mutexDir + when(mockProvider.readAttributes( + org.mockito.ArgumentMatchers.eq(mockMutexDir), + org.mockito.ArgumentMatchers.eq(java.nio.file.attribute.BasicFileAttributes.class), + org.mockito.ArgumentMatchers.any())) + .thenThrow(new IOException("Simulated I/O failure on mutex attributes")); + + // Should catch IOException and return false + assertFalse(lockingService.isLockDirectoryExpired(mockLockDir, System.currentTimeMillis())); } } diff --git a/src/test/java/me/desair/tus/server/upload/disk/LeaseFileUploadLockTest.java b/src/test/java/me/desair/tus/server/upload/disk/LeaseFileUploadLockTest.java index 97b662e5..98242cea 100644 --- a/src/test/java/me/desair/tus/server/upload/disk/LeaseFileUploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/disk/LeaseFileUploadLockTest.java @@ -19,6 +19,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import me.desair.tus.server.upload.LeaseData; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.LeaseDataJsonSerializer; import org.apache.commons.io.FileUtils; @@ -37,6 +38,7 @@ public class LeaseFileUploadLockTest { private static Path storagePath; private Path testLockDir; private Path testStopFile; + private final UploadId defaultUploadId = new UploadId("test-upload-id"); @BeforeClass public static void setupDataFolder() throws IOException { @@ -84,11 +86,12 @@ public void testHeartbeatRenewalUpdatesExpiresAt() throws Exception { String holderId = "holder-" + UUID.randomUUID(); String requestUri = "/files/upload/test-123"; long leaseDurationMs = 30_000L; - Map activeStreams = new ConcurrentHashMap<>(); + Map activeStreams = new ConcurrentHashMap<>(); LeaseData leaseData = createLeaseData(holderId, requestUri, leaseDurationMs); LeaseFileUploadLock lock = - new LeaseFileUploadLock(leaseData, testLockDir, testStopFile, activeStreams); + new LeaseFileUploadLock( + leaseData, defaultUploadId, testLockDir, testStopFile, activeStreams); long initialExpiresAt = lock.getExpiresAt(); Thread.sleep(50L); @@ -115,8 +118,8 @@ public void testCloseTerminatesHeartbeatAndDeletesLockDir() throws Exception { String holderId = "holder-" + UUID.randomUUID(); String requestUri = "/files/upload/test-456"; long leaseDurationMs = 30_000L; - Map activeStreams = new ConcurrentHashMap<>(); - activeStreams.put(requestUri, new ByteArrayInputStream("data".getBytes())); + Map activeStreams = new ConcurrentHashMap<>(); + activeStreams.put(defaultUploadId, new ByteArrayInputStream("data".getBytes())); // Create stop file to verify it gets cleaned up on close Files.write(testStopFile, new byte[0]); @@ -124,7 +127,8 @@ public void testCloseTerminatesHeartbeatAndDeletesLockDir() throws Exception { LeaseData leaseData = createLeaseData(holderId, requestUri, leaseDurationMs); LeaseFileUploadLock lock = - new LeaseFileUploadLock(leaseData, testLockDir, testStopFile, activeStreams); + new LeaseFileUploadLock( + leaseData, defaultUploadId, testLockDir, testStopFile, activeStreams); // Write initial lease file lock.renewLease(); @@ -141,7 +145,7 @@ public void testCloseTerminatesHeartbeatAndDeletesLockDir() throws Exception { assertFalse(Files.exists(testStopFile)); // Verify active stream was unregistered - assertFalse(activeStreams.containsKey(requestUri)); + assertFalse(activeStreams.containsKey(defaultUploadId)); } @Test @@ -149,11 +153,12 @@ public void testReleaseDelegatesToClose() throws Exception { String holderId = "holder-" + UUID.randomUUID(); String requestUri = "/files/upload/test-release"; long leaseDurationMs = 30_000L; - Map activeStreams = new ConcurrentHashMap<>(); + Map activeStreams = new ConcurrentHashMap<>(); LeaseData leaseData = createLeaseData(holderId, requestUri, leaseDurationMs); LeaseFileUploadLock lock = - new LeaseFileUploadLock(leaseData, testLockDir, testStopFile, activeStreams); + new LeaseFileUploadLock( + leaseData, defaultUploadId, testLockDir, testStopFile, activeStreams); assertThat(lock.getUploadUri(), is(requestUri)); @@ -191,7 +196,8 @@ public void testLeaseDataGettersAndSetters() { @Test public void testActiveLockConstructorInitializesFields() { LeaseData leaseData = createLeaseData("holder", "/uri", 10000L); - LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, testLockDir, testStopFile, null); + LeaseFileUploadLock lock = + new LeaseFileUploadLock(leaseData, defaultUploadId, testLockDir, testStopFile, null); assertThat(lock.getHolderId(), is("holder")); assertThat(lock.getRequestUri(), is("/uri")); @@ -207,7 +213,8 @@ public void testActiveLockConstructorInitializesFields() { @Test public void testConstructorWithZeroLeaseDurationDoesNotScheduleWatchdog() { LeaseData leaseData = createLeaseData("holder", "/uri", 0L); - LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, testLockDir, testStopFile, null); + LeaseFileUploadLock lock = + new LeaseFileUploadLock(leaseData, defaultUploadId, testLockDir, testStopFile, null); assertEquals("holder", lock.getHolderId()); lock.close(); } @@ -215,7 +222,8 @@ public void testConstructorWithZeroLeaseDurationDoesNotScheduleWatchdog() { @Test public void testRenewLeaseWithNullLockDirShouldBeNoOp() { LeaseData leaseData = createLeaseData("holder", "/uri", 10000L); - LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, null, null, null); + LeaseFileUploadLock lock = + new LeaseFileUploadLock(leaseData, defaultUploadId, null, null, null); // Should safely do nothing without throwing exception lock.renewLease(); assertEquals("holder", lock.getHolderId()); @@ -226,7 +234,8 @@ public void testRenewLeaseWithNullLockDirShouldBeNoOp() { public void testRenewLeaseWhenLockDirIsDeletedOrInvalid() { Path nonExistentDir = storagePath.resolve("non-existent-lock-dir-" + UUID.randomUUID()); LeaseData leaseData = createLeaseData("holder", "/uri", 10000L); - LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, nonExistentDir, null, null); + LeaseFileUploadLock lock = + new LeaseFileUploadLock(leaseData, defaultUploadId, nonExistentDir, null, null); // When directory doesn't exist, renewLease logs a warning and does not throw lock.renewLease(); assertEquals("holder", lock.getHolderId()); @@ -240,7 +249,7 @@ public void testCloseWhenLockDirIsAlreadyDeleted() throws Exception { FileUtils.deleteDirectory(dir.toFile()); LeaseData leaseData = createLeaseData("holder", "/uri", 10000L); - LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, dir, null, null); + LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, defaultUploadId, dir, null, null); // Should execute cleanly without error lock.close(); assertFalse(Files.exists(dir)); @@ -253,17 +262,18 @@ public void testCloseWithActiveStreamsAndStopFile() throws Exception { Path stopFile = storagePath.resolve("stop-file-" + UUID.randomUUID()); Files.write(stopFile, new byte[0]); - Map streams = new ConcurrentHashMap<>(); - streams.put("/active/uri", new ByteArrayInputStream("test".getBytes())); + Map streams = new ConcurrentHashMap<>(); + streams.put(defaultUploadId, new ByteArrayInputStream("test".getBytes())); LeaseData leaseData = createLeaseData("holder", "/active/uri", 10000L); - LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, dir, stopFile, streams); + LeaseFileUploadLock lock = + new LeaseFileUploadLock(leaseData, defaultUploadId, dir, stopFile, streams); assertTrue(Files.exists(stopFile)); lock.close(); assertFalse(Files.exists(stopFile)); - assertFalse(streams.containsKey("/active/uri")); + assertFalse(streams.containsKey(defaultUploadId)); } @Test @@ -271,17 +281,18 @@ public void testCloseWithNullRequestUriOrNullStreams() throws Exception { Path dir = storagePath.resolve("null-uri-" + UUID.randomUUID()); Files.createDirectories(dir); - Map streams = new ConcurrentHashMap<>(); + Map streams = new ConcurrentHashMap<>(); LeaseData leaseDataNullUri = createLeaseData("holder", null, 10000L); LeaseFileUploadLock lockWithNullUri = - new LeaseFileUploadLock(leaseDataNullUri, dir, null, streams); + new LeaseFileUploadLock(leaseDataNullUri, defaultUploadId, dir, null, streams); lockWithNullUri.close(); assertNotNull(lockWithNullUri.getHolderId()); Path dir2 = storagePath.resolve("null-streams-" + UUID.randomUUID()); Files.createDirectories(dir2); LeaseData leaseData = createLeaseData("holder", "/uri", 10000L); - LeaseFileUploadLock lockWithNullStreams = new LeaseFileUploadLock(leaseData, dir2, null, null); + LeaseFileUploadLock lockWithNullStreams = + new LeaseFileUploadLock(leaseData, null, dir2, null, null); lockWithNullStreams.close(); assertNotNull(lockWithNullStreams.getHolderId()); } @@ -295,7 +306,7 @@ public void testCloseWhenLockDirCannotBeDeleted() throws Exception { Files.write(dir.resolve("extra-file.txt"), "data".getBytes()); LeaseData leaseData = createLeaseData("holder", "/uri", 10000L); - LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, dir, null, null); + LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, defaultUploadId, dir, null, null); // Should catch DirectoryNotEmptyException, log warning, and complete cleanly lock.close(); assertTrue(Files.exists(dir)); @@ -309,7 +320,8 @@ public void testRenewLeaseWhenLockDirPathIsRegularFile() throws Exception { Files.write(fileAsDir, "not a directory".getBytes()); LeaseData leaseData = createLeaseData("holder", "/uri", 10000L); - LeaseFileUploadLock lock = new LeaseFileUploadLock(leaseData, fileAsDir, null, null); + LeaseFileUploadLock lock = + new LeaseFileUploadLock(leaseData, defaultUploadId, fileAsDir, null, null); // Should catch exception attempting to create file inside a regular file and log warning lock.renewLease(); assertTrue(Files.exists(fileAsDir)); @@ -344,7 +356,8 @@ public void testCloseDoesNotDeleteSuccessorLockWhenHolderIdMismatch() throws Exc System.currentTimeMillis() - 31_000L, dir.toString(), null); - LeaseFileUploadLock originalLock = new LeaseFileUploadLock(originalLease, dir, null, null); + LeaseFileUploadLock originalLock = + new LeaseFileUploadLock(originalLease, defaultUploadId, dir, null, null); originalLock.close(); @@ -378,10 +391,10 @@ public void testRenewLeaseAbortsWhenHolderIdMismatch() throws Exception { LeaseDataJsonSerializer.serializeToPath(successorLease, dir.resolve("lease.json")); // Stale original lock holder calls renewLease() with an active streaming request registered - Map activeStreams = new ConcurrentHashMap<>(); + Map activeStreams = new ConcurrentHashMap<>(); InterruptibleInputStream stream = new InterruptibleInputStream(new ByteArrayInputStream("data".getBytes())); - activeStreams.put("/files/upload/test", stream); + activeStreams.put(defaultUploadId, stream); LeaseData originalLease = new LeaseData( @@ -393,7 +406,7 @@ public void testRenewLeaseAbortsWhenHolderIdMismatch() throws Exception { dir.toString(), null); LeaseFileUploadLock originalLock = - new LeaseFileUploadLock(originalLease, dir, null, activeStreams); + new LeaseFileUploadLock(originalLease, defaultUploadId, dir, null, activeStreams); originalLock.renewLease(); @@ -414,8 +427,44 @@ public void testRenewLeaseAbortsWhenHolderIdMismatch() throws Exception { @Test public void testDoesLockOwnershipMatchWithNullLockDirPath() { LeaseData lease = new LeaseData("h", "/uri", 10000L, 50000L); - LeaseFileUploadLock lock = new LeaseFileUploadLock(lease, null, null, null); + LeaseFileUploadLock lock = new LeaseFileUploadLock(lease, defaultUploadId, null, null, null); + assertFalse(lock.doesLockOwnershipMatch()); + } + + @Test + public void testDoesLockOwnershipMatchWithMissingLeaseFile() throws Exception { + Path dir = storagePath.resolve("missing-lease-test-" + UUID.randomUUID() + ".lock"); + Files.createDirectories(dir); + + LeaseData lease = new LeaseData("h", "/uri", 10000L, 50000L, 40000L, dir.toString(), null); + LeaseFileUploadLock lock = new LeaseFileUploadLock(lease, defaultUploadId, dir, null, null); + // Missing lease.json returns false assertFalse(lock.doesLockOwnershipMatch()); + + DiskUtils.deleteDirectoryQuietly(dir); + } + + @Test + public void testDoesLockOwnershipMatchWithValidAndMismatchedHolder() throws Exception { + Path dir = storagePath.resolve("match-lease-test-" + UUID.randomUUID() + ".lock"); + Files.createDirectories(dir); + + LeaseData lease = new LeaseData("h", "/uri", 10000L, 50000L, 40000L, dir.toString(), null); + LeaseFileUploadLock lock = new LeaseFileUploadLock(lease, defaultUploadId, dir, null, null); + + // Matching holder returns true + LeaseData matchingLease = + new LeaseData("h", "/uri", 10000L, 50000L, 40000L, dir.toString(), null); + LeaseDataJsonSerializer.serializeToPath(matchingLease, dir.resolve("lease.json")); + assertTrue(lock.doesLockOwnershipMatch()); + + // Mismatched holder returns false + LeaseData mismatchedLease = + new LeaseData("different-holder", "/uri", 10000L, 50000L, 40000L, dir.toString(), null); + LeaseDataJsonSerializer.serializeToPath(mismatchedLease, dir.resolve("lease.json")); + assertFalse(lock.doesLockOwnershipMatch()); + + DiskUtils.deleteDirectoryQuietly(dir); } @Test @@ -425,9 +474,9 @@ public void testDoesLockOwnershipMatchWithCorruptedLeaseFile() throws Exception Files.write(dir.resolve("lease.json"), "invalid json".getBytes()); LeaseData lease = new LeaseData("h", "/uri", 10000L, 50000L, 40000L, dir.toString(), null); - LeaseFileUploadLock lock = new LeaseFileUploadLock(lease, dir, null, null); - // Unparseable or corrupt lease file should be considered owned / proceed - assertTrue(lock.doesLockOwnershipMatch()); + LeaseFileUploadLock lock = new LeaseFileUploadLock(lease, defaultUploadId, dir, null, null); + // Fail-closed split-brain protection: unparseable or corrupt lease file must return false + assertFalse(lock.doesLockOwnershipMatch()); FileUtils.deleteDirectory(dir.toFile()); } @@ -438,7 +487,7 @@ public void testDoRenewLeaseWhenMutexAcquisitionFails() throws Exception { Files.createDirectories(dir); LeaseData lease = new LeaseData("h", "/uri", 10000L, 50000L, 40000L, dir.toString(), null); - LeaseFileUploadLock lock = new LeaseFileUploadLock(lease, dir, null, null); + LeaseFileUploadLock lock = new LeaseFileUploadLock(lease, defaultUploadId, dir, null, null); // Pre-acquire the mutex to simulate another thread holding it try (LeaseFileMutex mutex = new LeaseFileMutex(dir)) { diff --git a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java index 2102dfe4..260dbc7d 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java +++ b/src/test/java/me/desair/tus/server/upload/s3/ITS3StorageService.java @@ -17,6 +17,8 @@ import java.nio.charset.StandardCharsets; import me.desair.tus.server.TestUtils; import me.desair.tus.server.checksum.ChecksumAlgorithm; +import me.desair.tus.server.checksum.ChecksumKey; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadInfo; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.IOUtils; @@ -266,6 +268,53 @@ public void testPruneOrphanedChecksumIndicesAbandonedTombstone() throws Exceptio assertFalse(objectExists(parentDeletedKey)); } + @Test + public void testS3ChecksumIndexDirectly() throws Exception { + S3ChecksumIndex index = new S3ChecksumIndex(minioClient, BUCKET, "test-checksums"); + ChecksumKey key = + ChecksumKey.of("1122334455667788990011223344556677889900", ChecksumAlgorithm.SHA1); + UploadId parentId = new UploadId("parent-upload-id"); + UploadId competingId = new UploadId("competing-upload-id"); + + // 1. Initial lookup returns null + assertNull(index.findParent(key)); + assertFalse(index.isParentDeleted(key)); + assertFalse(index.hasChildren(key)); + + // 2. First registration succeeds + assertTrue(index.tryRegisterParent(key, parentId)); + assertEquals(parentId, index.findParent(key)); + + // 3. Second registration fails due to CAS (precondition failed) or duplicate parent + assertFalse(index.tryRegisterParent(key, competingId)); + assertEquals(parentId, index.findParent(key)); + + // 4. Add and remove child + UploadId childId = new UploadId("child-upload-id"); + index.addChild(key, childId); + assertTrue(index.hasChildren(key)); + index.removeChild(key, childId); + assertFalse(index.hasChildren(key)); + + // 5. Mark parent deleted (preserves parent file for active children and new joiners) + index.markParentDeleted(key); + assertTrue(index.isParentDeleted(key)); + assertEquals(parentId, index.findParent(key)); + + // 6. Registration after tombstone returns false + assertFalse(index.tryRegisterParent(key, competingId)); + + // 7. Empty parent object returns null + ChecksumKey emptyKey = + ChecksumKey.of("2233445566778899001122334455667788990011", ChecksumAlgorithm.SHA1); + putObject("test-checksums/sha1/2233445566778899001122334455667788990011/parent", new byte[0]); + assertNull(index.findParent(emptyKey)); + + // 8. Delete entire index + index.deleteIndex(key); + assertNull(index.findParent(key)); + } + private void putObject(String objectKey, byte[] data) throws Exception { minioClient.putObject( PutObjectArgs.builder().bucket(BUCKET).object(objectKey).stream( diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java index 4d518f5e..40ac459a 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3ConcatenationServiceTest.java @@ -550,4 +550,80 @@ public void testSetS3ServerSideComposeHelper() { concatenationService.setS3ServerSideComposeHelper(mockHelper); assertNotNull(concatenationService); } + + @Test + public void testMergeResolvesObjectKeysWhenStorageServiceNotS3StorageService() throws Exception { + S3ServerSideComposeHelper mockHelper = Mockito.mock(S3ServerSideComposeHelper.class); + concatenationService.setS3ServerSideComposeHelper(mockHelper); + + UploadInfo childDup = new UploadInfo(); + childDup.setId(new UploadId("child-dup")); + childDup.setDuplicatesUploadId(new UploadId("parent-dup")); + childDup.setStorageUploadId("uploads/child-dup"); + childDup.setLength(5L * 1024 * 1024); + childDup.setOffset(5L * 1024 * 1024); + + UploadInfo childStorage = new UploadInfo(); + childStorage.setId(new UploadId("child-storage")); + childStorage.setStorageUploadId("custom/storage-path"); + childStorage.setLength(5L * 1024 * 1024); + childStorage.setOffset(5L * 1024 * 1024); + + UploadInfo parent = new UploadInfo(); + parent.setId(new UploadId("final-upload")); + parent.setConcatenationPartIds(Arrays.asList("/child-dup", "/child-storage")); + + Mockito.when(storageService.getUploadInfo("/child-dup", null)).thenReturn(childDup); + Mockito.when(storageService.getUploadInfo("/child-storage", null)).thenReturn(childStorage); + + concatenationService.merge(parent); + + Mockito.verify(mockHelper) + .compose( + Mockito.eq("test-bucket"), + Mockito.eq("uploads/final-upload"), + Mockito.eq(Arrays.asList("uploads/parent-dup", "custom/storage-path"))); + assertEquals(Long.valueOf(10L * 1024 * 1024), parent.getLength()); + assertEquals(Long.valueOf(10L * 1024 * 1024), parent.getOffset()); + } + + @Test + public void testMergeResolvesObjectKeysWhenStorageServiceIsS3StorageService() throws Exception { + S3StorageService mockS3Storage = Mockito.mock(S3StorageService.class); + S3ConcatenationService service = + new S3ConcatenationService( + minioClient, + "test-bucket", + "uploads/", + mockS3Storage, + Paths.get(System.getProperty("java.io.tmpdir")), + 5242880L, + null); + + S3ServerSideComposeHelper mockHelper = Mockito.mock(S3ServerSideComposeHelper.class); + service.setS3ServerSideComposeHelper(mockHelper); + + UploadInfo child1 = new UploadInfo(); + child1.setId(new UploadId("part-1")); + child1.setStorageUploadId("uploads/part-1"); + child1.setLength(5L * 1024 * 1024); + child1.setOffset(5L * 1024 * 1024); + + UploadInfo parent = new UploadInfo(); + parent.setId(new UploadId("final-upload")); + parent.setConcatenationPartIds(Collections.singletonList("/part-1")); + + Mockito.when(mockS3Storage.getUploadInfo("/part-1", null)).thenReturn(child1); + Mockito.when(mockS3Storage.getS3ObjectKey(child1)).thenReturn("resolved/s3-key-1"); + + service.merge(parent); + + Mockito.verify(mockHelper) + .compose( + Mockito.eq("test-bucket"), + Mockito.eq("uploads/final-upload"), + Mockito.eq(Collections.singletonList("resolved/s3-key-1"))); + assertEquals(Long.valueOf(5L * 1024 * 1024), parent.getLength()); + assertEquals(Long.valueOf(5L * 1024 * 1024), parent.getOffset()); + } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java index e38b4dcb..c39d20de 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3LockingServiceTest.java @@ -13,25 +13,44 @@ import io.minio.ListObjectsArgs; import io.minio.MinioClient; import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; import io.minio.Result; import io.minio.StatObjectArgs; +import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteRequest; import io.minio.messages.ErrorResponse; import io.minio.messages.Item; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; import me.desair.tus.server.exception.UploadAlreadyLockedException; import me.desair.tus.server.upload.LeaseData; +import me.desair.tus.server.upload.TimeBasedUploadIdFactory; import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.upload.UploadLock; import me.desair.tus.server.upload.UuidUploadIdFactory; import me.desair.tus.server.util.InterruptibleInputStream; +import me.desair.tus.server.util.LeaseDataJsonSerializer; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -60,16 +79,18 @@ public void setUp() throws Exception { invocation -> { PutObjectArgs args = invocation.getArgument(0); var headers = args.extraHeaders(); - if (headers != null - && "*".equals(headers.get("If-None-Match")) - && s3StorageMap.containsKey(args.object())) { - ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); - Mockito.when(errorResponse.code()).thenReturn("PreconditionFailed"); - throw new ErrorResponseException(errorResponse, null, "PreconditionFailed"); - } try (InputStream stream = args.stream()) { byte[] bytes = stream.readAllBytes(); - s3StorageMap.put(args.object(), bytes); + synchronized (s3StorageMap) { + if (headers != null + && "*".equals(headers.getFirst("If-None-Match")) + && s3StorageMap.containsKey(args.object())) { + ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(errorResponse.code()).thenReturn("PreconditionFailed"); + throw new ErrorResponseException(errorResponse, null, "PreconditionFailed"); + } + s3StorageMap.put(args.object(), bytes); + } } return null; }); @@ -78,7 +99,10 @@ public void setUp() throws Exception { .thenAnswer( invocation -> { GetObjectArgs args = invocation.getArgument(0); - byte[] bytes = s3StorageMap.get(args.object()); + byte[] bytes; + synchronized (s3StorageMap) { + bytes = s3StorageMap.get(args.object()); + } if (bytes == null) { ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); Mockito.when(errorResponse.code()).thenReturn("NoSuchKey"); @@ -94,21 +118,22 @@ public void setUp() throws Exception { Mockito.doAnswer( invocation -> { - io.minio.RemoveObjectArgs args = invocation.getArgument(0); - s3StorageMap.remove(args.object()); + RemoveObjectArgs args = invocation.getArgument(0); + synchronized (s3StorageMap) { + s3StorageMap.remove(args.object()); + } return null; }) .when(minioClient) - .removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); + .removeObject(Mockito.any(RemoveObjectArgs.class)); Mockito.doAnswer( invocation -> { - io.minio.RemoveObjectsArgs args = invocation.getArgument(0); + RemoveObjectsArgs args = invocation.getArgument(0); if (args != null && args.objects() != null) { - for (io.minio.messages.DeleteRequest.Object obj : args.objects()) { + for (DeleteRequest.Object obj : args.objects()) { try { - java.lang.reflect.Field field = - io.minio.messages.DeleteRequest.Object.class.getDeclaredField("name"); + Field field = DeleteRequest.Object.class.getDeclaredField("name"); field.setAccessible(true); String name = (String) field.get(obj); s3StorageMap.remove(name); @@ -116,10 +141,10 @@ public void setUp() throws Exception { } } } - return java.util.Collections.emptyList(); + return Collections.emptyList(); }) .when(minioClient) - .removeObjects(Mockito.any(io.minio.RemoveObjectsArgs.class)); + .removeObjects(Mockito.any(RemoveObjectsArgs.class)); } @Test @@ -165,7 +190,7 @@ public void testExplicitConnectionParametersConstructors() { "locks/", 30000L, 2000L, - new me.desair.tus.server.upload.UuidUploadIdFactory()); + new UuidUploadIdFactory()); assertNotNull(fullServiceWithParams); S3LockingService defaultRegionService = @@ -206,16 +231,17 @@ public void testIsLocked() throws Exception { } @Test - public void testIsLockedReturnsFalseOnGenericMinioException() throws Exception { + public void testIsLockedReturnsTrueOnGenericMinioExceptionFailClosed() throws Exception { Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))) .thenThrow(new RuntimeException("GetObject failure")); UploadId uploadId = new UploadId("24249a5b-01a4-4bf8-b67a-364273bb5a2e"); - assertFalse(lockingService.isLocked(uploadId)); + // Fail-closed split-brain protection: transient error assumes lock is still active + assertTrue(lockingService.isLocked(uploadId)); } @Test - public void testIsLockedReturnsFalseOnErrorResponseNon404() throws Exception { + public void testIsLockedReturnsTrueOnErrorResponseNon404FailClosed() throws Exception { ErrorResponse errorResponse = Mockito.mock(ErrorResponse.class); Mockito.when(errorResponse.code()).thenReturn("AccessDenied"); ErrorResponseException accessDeniedEx = new ErrorResponseException(errorResponse, null, null); @@ -223,7 +249,8 @@ public void testIsLockedReturnsFalseOnErrorResponseNon404() throws Exception { Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))).thenThrow(accessDeniedEx); UploadId uploadId = new UploadId("24249a5b-01a4-4bf8-b67a-364273bb5a2e"); - assertFalse(lockingService.isLocked(uploadId)); + // Fail-closed split-brain protection: non-404 S3 error assumes lock is still active + assertTrue(lockingService.isLocked(uploadId)); } @Test @@ -240,7 +267,7 @@ public void testLockAcquisitionFailures() throws Exception { @Test public void testRegisterInputStreamAndRequestLockReleaseWithInterruption() throws Exception { - lockingService.setIdFactory(new me.desair.tus.server.upload.UuidUploadIdFactory()); + lockingService.setIdFactory(new UuidUploadIdFactory()); // Test with InterruptibleInputStream InterruptibleInputStream interruptibleStream = Mockito.mock(InterruptibleInputStream.class); @@ -250,7 +277,7 @@ public void testRegisterInputStreamAndRequestLockReleaseWithInterruption() throw // Mock statObject to succeed (stop key exists) Mockito.when(minioClient.statObject(Mockito.any(StatObjectArgs.class))) - .thenReturn(Mockito.mock(io.minio.StatObjectResponse.class)); + .thenReturn(Mockito.mock(StatObjectResponse.class)); lockingService.requestLockRelease("/files/upload/24249a5b-01a4-4bf8-b67a-364273bb5a2e"); Mockito.verify(interruptibleStream).interrupt(); @@ -271,7 +298,7 @@ public void testCleanupStaleLocksWithExpiredItem() throws Exception { Mockito.when(item.objectName()).thenReturn("tus-locks/expired.lock"); Result result = new Result<>(item); - Iterable> results = java.util.Collections.singletonList(result); + Iterable> results = Collections.singletonList(result); Mockito.when(minioClient.listObjects(Mockito.any(ListObjectsArgs.class))).thenReturn(results); @@ -291,6 +318,39 @@ public void testCleanupStaleLocksWithExpiredItem() throws Exception { lockingService.cleanupStaleLocks(); } + @Test + public void testCleanupStaleLocksHandlesNullItemAndStaleStopSignal() throws Exception { + Item nullNameItem = Mockito.mock(Item.class); + Mockito.when(nullNameItem.objectName()).thenReturn(null); + + Item staleStopItem = Mockito.mock(Item.class); + Mockito.when(staleStopItem.objectName()).thenReturn("tus-locks/stale.stop"); + Mockito.when(staleStopItem.lastModified()).thenReturn(ZonedDateTime.now().minusMinutes(2)); + + Item recentStopItem = Mockito.mock(Item.class); + Mockito.when(recentStopItem.objectName()).thenReturn("tus-locks/recent.stop"); + Mockito.when(recentStopItem.lastModified()).thenReturn(ZonedDateTime.now()); + + java.util.List> items = + Arrays.asList( + new Result<>(null), + new Result<>(nullNameItem), + new Result<>(staleStopItem), + new Result<>(recentStopItem)); + Mockito.when(minioClient.listObjects(Mockito.any(ListObjectsArgs.class))).thenReturn(items); + + lockingService.cleanupStaleLocks(); + + Mockito.verify(minioClient) + .removeObject( + Mockito.argThat( + (RemoveObjectArgs a) -> a != null && "tus-locks/stale.stop".equals(a.object()))); + Mockito.verify(minioClient, Mockito.never()) + .removeObject( + Mockito.argThat( + (RemoveObjectArgs a) -> a != null && "tus-locks/recent.stop".equals(a.object()))); + } + @Test(expected = IOException.class) public void testCleanupStaleLocksThrowsIOExceptionOnMinioFailure() throws Exception { Mockito.when(minioClient.listObjects(Mockito.any(ListObjectsArgs.class))) @@ -545,7 +605,7 @@ public void testCleanupStaleLocksWithExpiredAndNonExpiredLocks() throws Exceptio service.cleanupStaleLocks(); // Expired lock object is deleted - Mockito.verify(mockClient).removeObject(Mockito.any(io.minio.RemoveObjectArgs.class)); + Mockito.verify(mockClient).removeObject(Mockito.any(RemoveObjectArgs.class)); } @Test @@ -561,8 +621,7 @@ public void testWriteStopSignalException() throws Exception { .thenThrow(new RuntimeException("S3 Put error")); S3LockingService service = createLockingService(mockClient, "test-bucket"); - me.desair.tus.server.upload.TimeBasedUploadIdFactory idFactory = - new me.desair.tus.server.upload.TimeBasedUploadIdFactory(); + TimeBasedUploadIdFactory idFactory = new TimeBasedUploadIdFactory(); idFactory.setUploadUri("/files/upload"); service.setIdFactory(idFactory); @@ -614,6 +673,8 @@ public void testConditionalWriteConflictThrowsUploadAlreadyLockedException() thr @Test(expected = UploadAlreadyLockedException.class) public void testLockUploadByUriWithContentionOnPostPutVerificationThrows() throws Exception { + lockingService.withS3ConditionalWritesSupported(false); + // Simulate a TOCTOU race where PutObject succeeds, but another contender overwrote the lock // before our read-after-write verification Mockito.when(minioClient.putObject(Mockito.any(PutObjectArgs.class))).thenReturn(null); @@ -627,7 +688,7 @@ public void testLockUploadByUriWithContentionOnPostPutVerificationThrows() throw System.currentTimeMillis(), "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.lock", "locks/24249a5b-01a4-4bf8-b67a-364273bb5a2e.stop"); - String rivalJson = me.desair.tus.server.util.LeaseDataJsonSerializer.serialize(rivalLock); + String rivalJson = LeaseDataJsonSerializer.serialize(rivalLock); // PutObject succeeds, but post-put read-after-write verification sees rival holder Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))) @@ -657,15 +718,12 @@ public void testThreadContentionSingleWinnerS3() throws Exception { String uri = "/files/upload/" + uploadIdStr; int threadCount = 15; - java.util.concurrent.ExecutorService executor = - java.util.concurrent.Executors.newFixedThreadPool(threadCount); - java.util.List> tasks = new java.util.ArrayList<>(); - java.util.concurrent.atomic.AtomicInteger successCount = - new java.util.concurrent.atomic.AtomicInteger(0); - java.util.concurrent.atomic.AtomicInteger lockedExceptionCount = - new java.util.concurrent.atomic.AtomicInteger(0); - java.util.List acquiredLocks = new java.util.concurrent.CopyOnWriteArrayList<>(); - java.util.concurrent.CountDownLatch startLatch = new java.util.concurrent.CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + List> tasks = new ArrayList<>(); + AtomicInteger successCount = new AtomicInteger(0); + AtomicInteger lockedExceptionCount = new AtomicInteger(0); + List acquiredLocks = new CopyOnWriteArrayList<>(); + CountDownLatch startLatch = new CountDownLatch(1); for (int i = 0; i < threadCount; i++) { tasks.add( @@ -714,20 +772,16 @@ public void testConcurrentEvictionContentionS3() throws Exception { System.currentTimeMillis() - 40_000L, lockKey, "locks/" + uploadIdStr + ".stop"); - byte[] expiredBytes = - me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(expiredLock); + byte[] expiredBytes = LeaseDataJsonSerializer.serializeToBytes(expiredLock); s3StorageMap.put(lockKey, expiredBytes); int threadCount = 10; - java.util.concurrent.ExecutorService executor = - java.util.concurrent.Executors.newFixedThreadPool(threadCount); - java.util.List> tasks = new java.util.ArrayList<>(); - java.util.concurrent.atomic.AtomicInteger successCount = - new java.util.concurrent.atomic.AtomicInteger(0); - java.util.concurrent.atomic.AtomicInteger lockedExceptionCount = - new java.util.concurrent.atomic.AtomicInteger(0); - java.util.List acquiredLocks = new java.util.concurrent.CopyOnWriteArrayList<>(); - java.util.concurrent.CountDownLatch startLatch = new java.util.concurrent.CountDownLatch(1); + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + List> tasks = new ArrayList<>(); + AtomicInteger successCount = new AtomicInteger(0); + AtomicInteger lockedExceptionCount = new AtomicInteger(0); + List acquiredLocks = new CopyOnWriteArrayList<>(); + CountDownLatch startLatch = new CountDownLatch(1); for (int i = 0; i < threadCount; i++) { tasks.add( @@ -806,10 +860,37 @@ public void testIsLockExpiredWhenErrorResponseExceptionNotNoSuchKey() throws Exc new ErrorResponseException(errorResponse, null, "InternalError"); Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))).thenThrow(internalEx); + // Fail-closed: non-NoSuchKey error assumes lock is still active (not expired) + boolean expired = lockingService.isLockExpired("locks/test.lock"); + assertFalse(expired); + } + + @Test + public void testIsLockExpiredWhenJsonCorruptedReturnsTrue() throws Exception { + byte[] corruptedBytes = "{ corrupt json ]".getBytes(StandardCharsets.UTF_8); + GetObjectResponse response = + new GetObjectResponse( + null, + "test-bucket", + "eu-central-1", + "locks/test.lock", + new ByteArrayInputStream(corruptedBytes)); + Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))).thenReturn(response); + boolean expired = lockingService.isLockExpired("locks/test.lock"); assertTrue(expired); } + @Test + public void testIsLockExpiredWhenGenericExceptionReturnsFalseFailClosed() throws Exception { + Mockito.when(minioClient.getObject(Mockito.any(GetObjectArgs.class))) + .thenThrow(new RuntimeException("Network down")); + + // Fail-closed: transient exception assumes lock is still active (not expired) + boolean expired = lockingService.isLockExpired("locks/test.lock"); + assertFalse(expired); + } + @Test public void testIsLockExpiredClockSkewSafetyMargin() throws Exception { long now = System.currentTimeMillis(); @@ -869,8 +950,7 @@ public void testEvictExpiredLockDeletesS3Object() throws Exception { Mockito.verify(minioClient, Mockito.times(1)) .removeObject( Mockito.argThat( - (io.minio.RemoveObjectArgs args) -> - "locks/test-expired-upload.lock".equals(args.object()))); + (RemoveObjectArgs args) -> "locks/test-expired-upload.lock".equals(args.object()))); } @Test @@ -986,10 +1066,10 @@ public void testConditionalWriteNotImplementedDowngradesToNonCasModeAndSucceeds( throw notImplementedEx; } // Unconditional fallback write stores bytes in our in-memory map - java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); byte[] buf = new byte[1024]; int read; - java.io.InputStream is = args.stream(); + InputStream is = args.stream(); while ((read = is.read(buf)) != -1) { baos.write(buf, 0, read); } @@ -1059,80 +1139,222 @@ public void testExplicitNonCasModeChecksLockExpirationBeforeUnconditionalWrite() } @Test - public void testCleanupLockNullAndInvalid() throws Exception { - lockingService.cleanupLock((UploadId) null); - lockingService.cleanupLock((String) null); - // KISS: verifying method executes cleanly without throwing an exception - assertTrue(true); + public void testLockCloseDeletesLockObject() throws Exception { + UploadId id = new UploadId(UUID.randomUUID()); + String uri = "/files/upload/" + id; + String lockKey = "locks/" + id + ".lock"; + + UploadLock lock = lockingService.lockUploadByUri(uri); + assertNotNull(lock); + assertTrue(s3StorageMap.containsKey(lockKey)); + + lock.close(); + assertFalse(s3StorageMap.containsKey(lockKey)); } @Test - public void testCleanupLockDeletesExpiredLockAndStopObjects() throws Exception { + public void testCleanupLockClearsStopSignal() throws Exception { UploadId id = new UploadId(UUID.randomUUID()); - String lockKey = "locks/" + id + ".lock"; String stopKey = "locks/" + id + ".stop"; + s3StorageMap.put(stopKey, new byte[0]); + + lockingService.cleanupLock(null); + lockingService.cleanupLock(id); + assertFalse(s3StorageMap.containsKey(stopKey)); + } + + @Test + public void testS3LockingServiceThreeArgConstructor() { + S3LockingService service = new S3LockingService(minioClient, "test-bucket", "locks/"); + assertNotNull(service); + } + + @Test + public void testTryAcquireLockDirectAndNullLeaseData() throws Exception { + UploadId uploadId = new UploadId(UUID.randomUUID()); + assertNull(lockingService.tryAcquireLock(uploadId, null)); + + LeaseData leaseData = + new LeaseData( + "holder", + "/files/upload/test", + 30000L, + System.currentTimeMillis() + 30000L, + System.currentTimeMillis(), + null, + null); + UploadLock lock = lockingService.tryAcquireLock(uploadId, leaseData); + assertNotNull(lock); + lock.close(); + } + + @Test + public void testCheckStopSignalForUploadNullAndException() throws Exception { + lockingService.checkStopSignalForUpload(null, Mockito.mock(InputStream.class)); + + UploadId id = new UploadId(UUID.randomUUID()); + Mockito.when(minioClient.statObject(Mockito.any(StatObjectArgs.class))) + .thenThrow(new RuntimeException("Network error")); + lockingService.checkStopSignalForUpload(id, Mockito.mock(InputStream.class)); + } + + @Test + public void testEvictExpiredLockStaleMutexRecovery() throws Exception { + UploadId uploadId = new UploadId("test-stale-mutex"); + String lockKey = "locks/" + uploadId + ".lock"; + String mutexKey = "locks/" + uploadId + ".mutex"; LeaseData expiredLock = new LeaseData( - "old-holder", - "/files/upload/" + id, + "expired-holder", + "/files/upload/" + uploadId, 30000L, - System.currentTimeMillis() - 5000L, - System.currentTimeMillis() - 35000L, + System.currentTimeMillis() - 10_000L, + System.currentTimeMillis() - 40_000L, lockKey, - stopKey); - s3StorageMap.put( - lockKey, me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(expiredLock)); - s3StorageMap.put(stopKey, new byte[0]); - - assertTrue(s3StorageMap.containsKey(lockKey)); - assertTrue(s3StorageMap.containsKey(stopKey)); + null); + s3StorageMap.put(lockKey, LeaseDataJsonSerializer.serializeToBytes(expiredLock)); - lockingService.cleanupLock(id); + long staleTime = System.currentTimeMillis() - 10_000L; + s3StorageMap.put(mutexKey, String.valueOf(staleTime).getBytes(StandardCharsets.UTF_8)); + boolean evicted = lockingService.evictExpiredLock(uploadId); + assertTrue(evicted); assertFalse(s3StorageMap.containsKey(lockKey)); - assertFalse(s3StorageMap.containsKey(stopKey)); + assertFalse(s3StorageMap.containsKey(mutexKey)); } @Test - public void testCleanupLockByUri() throws Exception { - UploadId id = new UploadId(UUID.randomUUID()); - String uri = "/files/upload/" + id; - String lockKey = "locks/" + id + ".lock"; + public void testEvictExpiredLockApiNotImplementedDowngradesToNonCas() throws Exception { + UploadId uploadId = new UploadId("test-non-cas-downgrade"); + String lockKey = "locks/" + uploadId + ".lock"; LeaseData expiredLock = new LeaseData( - "old-holder", - uri, + "expired-holder", + "/files/upload/" + uploadId, 30000L, - System.currentTimeMillis() - 5000L, - System.currentTimeMillis() - 35000L, + System.currentTimeMillis() - 10_000L, + System.currentTimeMillis() - 40_000L, lockKey, null); - s3StorageMap.put( - lockKey, me.desair.tus.server.util.LeaseDataJsonSerializer.serializeToBytes(expiredLock)); - assertTrue(s3StorageMap.containsKey(lockKey)); + s3StorageMap.put(lockKey, LeaseDataJsonSerializer.serializeToBytes(expiredLock)); - lockingService.cleanupLock(uri); + ErrorResponse notImplResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(notImplResponse.code()).thenReturn("NotImplemented"); + ErrorResponseException notImplEx = + new ErrorResponseException(notImplResponse, null, "NotImplemented"); - assertFalse(s3StorageMap.containsKey(lockKey)); + Mockito.doThrow(notImplEx) + .doAnswer( + invocation -> { + PutObjectArgs args = invocation.getArgument(0); + try (InputStream stream = args.stream()) { + s3StorageMap.put(args.object(), stream.readAllBytes()); + } + return null; + }) + .when(minioClient) + .putObject( + Mockito.argThat( + (PutObjectArgs args) -> + args != null && args.object().equals("locks/" + uploadId + ".mutex"))); + + boolean evicted = lockingService.evictExpiredLock(uploadId); + assertTrue(evicted); } @Test - public void testCleanupLockPreservesActivelyHeldLock() throws Exception { - UploadId id = new UploadId(UUID.randomUUID()); - String uri = "/files/upload/" + id; - String lockKey = "locks/" + id + ".lock"; + public void testEvictExpiredLockNonCasActiveMutexReturnsFalse() throws Exception { + lockingService.withS3ConditionalWritesSupported(false); + UploadId uploadId = new UploadId("test-non-cas-active"); + String mutexKey = "locks/" + uploadId + ".mutex"; - UploadLock lock = lockingService.lockUploadByUri(uri); - assertNotNull(lock); - assertTrue(s3StorageMap.containsKey(lockKey)); + long activeTime = System.currentTimeMillis(); + s3StorageMap.put(mutexKey, String.valueOf(activeTime).getBytes(StandardCharsets.UTF_8)); - // Actively held unexpired lock must not be deleted by cleanupLock - lockingService.cleanupLock(id); - assertTrue(s3StorageMap.containsKey(lockKey)); + boolean evicted = lockingService.evictExpiredLock(uploadId); + assertFalse(evicted); + } - lock.close(); - assertFalse(s3StorageMap.containsKey(lockKey)); + @Test + public void testEvictExpiredLockNonCasPutExceptionReturnsFalse() throws Exception { + lockingService.withS3ConditionalWritesSupported(false); + UploadId uploadId = new UploadId("test-non-cas-put-err"); + + Mockito.doThrow(new RuntimeException("Disk full")) + .when(minioClient) + .putObject( + Mockito.argThat( + (PutObjectArgs args) -> + args != null && args.object().equals("locks/" + uploadId + ".mutex"))); + + boolean evicted = lockingService.evictExpiredLock(uploadId); + assertFalse(evicted); + } + + @Test + public void testHighConcurrencyEvictionContentionS3() throws Exception { + String uploadIdStr = UUID.randomUUID().toString(); + String uri = "/files/upload/" + uploadIdStr; + + // Seed an expired lock in simulated S3 storage + String lockKey = "locks/" + uploadIdStr + ".lock"; + LeaseData expiredLock = + new LeaseData( + "expired-holder", + uri, + 30000L, + System.currentTimeMillis() - 10_000L, + System.currentTimeMillis() - 40_000L, + lockKey, + "locks/" + uploadIdStr + ".stop"); + byte[] expiredBytes = LeaseDataJsonSerializer.serializeToBytes(expiredLock); + s3StorageMap.put(lockKey, expiredBytes); + + int threadCount = 40; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + List> tasks = new ArrayList<>(); + AtomicInteger successCount = new AtomicInteger(0); + AtomicInteger lockedExceptionCount = new AtomicInteger(0); + List acquiredLocks = new CopyOnWriteArrayList<>(); + CountDownLatch startLatch = new CountDownLatch(1); + + for (int i = 0; i < threadCount; i++) { + tasks.add( + () -> { + try { + startLatch.await(); + UploadLock lock = lockingService.lockUploadByUri(uri); + if (lock != null) { + successCount.incrementAndGet(); + acquiredLocks.add(lock); + return true; + } + } catch (UploadAlreadyLockedException e) { + lockedExceptionCount.incrementAndGet(); + } + return false; + }); + } + + startLatch.countDown(); + executor.invokeAll(tasks); + executor.shutdown(); + + assertEquals(1, successCount.get()); + assertEquals(threadCount - 1, lockedExceptionCount.get()); + assertEquals(1, acquiredLocks.size()); + + for (UploadLock lock : acquiredLocks) { + lock.close(); + } + } + + @Test + public void testDeleteS3ObjectsQuietlyViaService() { + lockingService.deleteS3ObjectsQuietly(null); + lockingService.deleteS3ObjectsQuietly(Collections.emptyList()); + lockingService.deleteS3ObjectsQuietly(Collections.singletonList("locks/test.key")); } } diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java index 905dfc84..fa9bf213 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3StorageServiceTest.java @@ -12,6 +12,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -27,6 +28,7 @@ import io.minio.StatObjectArgs; import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteResult; import io.minio.messages.ErrorResponse; import io.minio.messages.Item; import java.io.ByteArrayInputStream; @@ -42,6 +44,11 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; import me.desair.tus.server.checksum.ChecksumAlgorithm; import me.desair.tus.server.checksum.ChecksumKey; import me.desair.tus.server.exception.MaxAppendSizeExceededException; @@ -665,6 +672,10 @@ public void testAppendCompletingUploadWithLeftoverPart() throws Exception { info.setId(new UploadId("24249a5b-01a4-4bf8-b67a-364273bb5a2e")); info.setLength(100L); info.setOffset(50L); + info.setUploadPartKeys( + new ArrayList<>( + Collections.singletonList( + "metadata/24249a5b-01a4-4bf8-b67a-364273bb5a2e.part.00001_50"))); String jsonBefore = UploadInfoJsonSerializer.serialize(info); info.setOffset(100L); @@ -1103,7 +1114,6 @@ public void testCleanupExpiredUploads() throws Exception { when(mockLocking.isLocked(expiredId)).thenReturn(false); storageService.cleanupExpiredUploads(mockLocking); - verify(mockLocking).cleanupLock(expiredId); } @Test @@ -1131,7 +1141,6 @@ public void testCleanupExpiredUploadsNoArg() throws Exception { storageService.setUploadLockingService(mockLocking); storageService.cleanupExpiredUploads(); - verify(mockLocking).cleanupLock(expiredId); } @Test(expected = IOException.class) @@ -1455,9 +1464,13 @@ public void testUploadChunkToS3ThrowsIOException() throws Exception { @Test(expected = IOException.class) public void testFinalizeCompletedUploadSinglePartComposeException() throws Exception { UploadInfo info = new UploadInfo(); + long fiveMb = 5L * 1024L * 1024L; info.setId(new UploadId("single-compose-err")); - info.setLength(10L); - info.setOffset(0L); + info.setLength(fiveMb); + info.setOffset(fiveMb); + info.setUploadPartKeys( + new ArrayList<>( + Collections.singletonList("metadata/single-compose-err.part.00001_" + fiveMb))); ErrorResponse noSuchKeyErr = mock(ErrorResponse.class); when(noSuchKeyErr.code()).thenReturn("NoSuchKey"); @@ -1474,7 +1487,7 @@ public void testFinalizeCompletedUploadSinglePartComposeException() throws Excep }); StatObjectResponse statRes = mock(StatObjectResponse.class); - when(statRes.size()).thenReturn(10L); + when(statRes.size()).thenReturn(fiveMb); when(minioClient.statObject(any(StatObjectArgs.class))) .thenAnswer( @@ -1487,7 +1500,7 @@ public void testFinalizeCompletedUploadSinglePartComposeException() throws Excep }); Item item1 = mock(Item.class); - when(item1.objectName()).thenReturn("uploads/single-compose-err.part.00001"); + when(item1.objectName()).thenReturn("uploads/single-compose-err.part.00001_" + fiveMb); when(minioClient.listObjects(any(ListObjectsArgs.class))) .thenReturn(Arrays.asList(new Result<>(item1))); @@ -1495,15 +1508,22 @@ public void testFinalizeCompletedUploadSinglePartComposeException() throws Excep .when(minioClient) .composeObject(any(io.minio.ComposeObjectArgs.class)); - storageService.append(info, new ByteArrayInputStream(new byte[10])); + storageService.append(info, new ByteArrayInputStream(new byte[0])); } @Test(expected = IOException.class) public void testFinalizeCompletedUploadMultipartComposeException() throws Exception { UploadInfo info = new UploadInfo(); + long fiveMb = 5L * 1024L * 1024L; + long totalLength = fiveMb * 2; info.setId(new UploadId("multi-compose-err")); - info.setLength(20L); - info.setOffset(0L); + info.setLength(totalLength); + info.setOffset(totalLength); + info.setUploadPartKeys( + new ArrayList<>( + Arrays.asList( + "metadata/multi-compose-err.part.00001_" + fiveMb, + "metadata/multi-compose-err.part.00002_" + fiveMb))); ErrorResponse noSuchKeyErr = mock(ErrorResponse.class); when(noSuchKeyErr.code()).thenReturn("NoSuchKey"); @@ -1520,7 +1540,7 @@ public void testFinalizeCompletedUploadMultipartComposeException() throws Except }); StatObjectResponse statRes = mock(StatObjectResponse.class); - when(statRes.size()).thenReturn(10L); + when(statRes.size()).thenReturn(fiveMb); when(minioClient.statObject(any(StatObjectArgs.class))) .thenAnswer( @@ -1533,9 +1553,9 @@ public void testFinalizeCompletedUploadMultipartComposeException() throws Except }); Item item1 = mock(Item.class); - when(item1.objectName()).thenReturn("uploads/multi-compose-err.part.00001"); + when(item1.objectName()).thenReturn("uploads/multi-compose-err.part.00001_" + fiveMb); Item item2 = mock(Item.class); - when(item2.objectName()).thenReturn("uploads/multi-compose-err.part.00002"); + when(item2.objectName()).thenReturn("uploads/multi-compose-err.part.00002_" + fiveMb); when(minioClient.listObjects(any(ListObjectsArgs.class))) .thenReturn(Arrays.asList(new Result<>(item1), new Result<>(item2))); @@ -1544,7 +1564,7 @@ public void testFinalizeCompletedUploadMultipartComposeException() throws Except .when(minioClient) .composeObject(any(ComposeObjectArgs.class)); - storageService.append(info, new ByteArrayInputStream(new byte[20])); + storageService.append(info, new ByteArrayInputStream(new byte[0])); } @Test(expected = IOException.class) @@ -2074,6 +2094,51 @@ public void testCleanupExpiredUploadsPreservesCompletedUpload() throws Exception (io.minio.RemoveObjectArgs args) -> args.object().equals("uploads/completed-123"))); } + @Test + public void testCleanupExpiredUploadsWithNullInfoCleansLock() throws Exception { + Item item = mock(Item.class); + when(item.objectName()).thenReturn("metadata/corrupt-456.info"); + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenReturn(java.util.Collections.singletonList(new Result<>(item))); + + ErrorResponse errorResponse = + new ErrorResponse("NoSuchKey", "Not found", null, null, null, null, null); + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenThrow(new ErrorResponseException(errorResponse, null, null)); + + UploadLockingService mockLocking = mock(UploadLockingService.class); + storageService.cleanupExpiredUploads(mockLocking); + + verify(mockLocking).cleanupLock(new UploadId("corrupt-456")); + } + + @Test + public void testCleanupExpiredUploadsCleansLockForExpiredInProgressUpload() throws Exception { + UploadInfo expiredInfo = new UploadInfo(); + UploadId id = new UploadId("expired-456"); + expiredInfo.setId(id); + expiredInfo.setLength(100L); + expiredInfo.setOffset(50L); + expiredInfo.setExpirationTimestamp(System.currentTimeMillis() - 10000L); + + String json = UploadInfoJsonSerializer.serialize(expiredInfo); + + Item item = mock(Item.class); + when(item.objectName()).thenReturn("metadata/expired-456.info"); + when(minioClient.listObjects(any(ListObjectsArgs.class))) + .thenReturn(java.util.Collections.singletonList(new Result<>(item))); + + when(minioClient.getObject(any(GetObjectArgs.class))) + .thenAnswer(invocation -> mockGetObjectResponse(json.getBytes())); + + UploadLockingService mockLocking = mock(UploadLockingService.class); + when(mockLocking.isLocked(id)).thenReturn(false); + + storageService.cleanupExpiredUploads(mockLocking); + + verify(mockLocking).cleanupLock(id); + } + @Test public void testCleanupExpiredUploadsPrunesStaleTempFilesAndChecksumIndices() throws Exception { java.nio.file.Path tempDir = java.nio.file.Files.createTempDirectory("s3-cleanup-test"); @@ -2359,16 +2424,14 @@ public void testRemoveLastNumberOfBytesWithManifestParts() throws Exception { @Test public void testDeleteS3ObjectsBatchFailureFallback() throws Exception { - assertTrue(storageService.isSupportsBatchDelete()); - UploadId uploadId = new UploadId("batch-del-fallback"); List keys = Arrays.asList("metadata/batch.part.00001", "metadata/batch.part.00002"); - io.minio.messages.DeleteResult.Error delErr = mock(io.minio.messages.DeleteResult.Error.class); + DeleteResult.Error delErr = mock(DeleteResult.Error.class); when(delErr.objectName()).thenReturn("metadata/batch.part.00001"); when(delErr.message()).thenReturn("AccessDenied"); @SuppressWarnings("unchecked") - Result errResult = mock(Result.class); + Result errResult = mock(Result.class); when(errResult.get()).thenReturn(delErr); when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) @@ -2382,25 +2445,10 @@ public void testDeleteS3ObjectsBatchFailureFallback() throws Exception { verify(minioClient) .removeObject( argThat((RemoveObjectArgs args) -> args.object().equals("metadata/batch.part.00002"))); - assertFalse(storageService.isSupportsBatchDelete()); - - // Subsequent batch delete should bypass removeObjects directly - storageService.deleteS3ObjectsQuietly(Collections.singletonList("metadata/batch.part.00003")); - verify(minioClient) - .removeObject( - argThat((RemoveObjectArgs args) -> args.object().equals("metadata/batch.part.00003"))); - // removeObjects was only called once (for the first attempt) - org.mockito.Mockito.verify(minioClient, org.mockito.Mockito.times(1)) - .removeObjects(any(RemoveObjectsArgs.class)); - - // Reset flag for subsequent tests - storageService.setSupportsBatchDelete(true); } @Test public void testDeleteS3ObjectsThrowsExceptionFallsBackToIndividualDelete() throws Exception { - assertTrue(storageService.isSupportsBatchDelete()); - UploadId uploadId = new UploadId("batch-del-ex"); List keys = Collections.singletonList("metadata/batch-ex.part.00001"); when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) @@ -2412,15 +2460,11 @@ public void testDeleteS3ObjectsThrowsExceptionFallsBackToIndividualDelete() thro .removeObject( argThat( (RemoveObjectArgs args) -> args.object().equals("metadata/batch-ex.part.00001"))); - assertFalse(storageService.isSupportsBatchDelete()); // Null or empty handling storageService.deleteS3ObjectsQuietly(null); storageService.deleteS3ObjectsQuietly(Collections.emptyList()); storageService.deleteS3ObjectsQuietly(Collections.singletonList(null)); - - // Reset flag - storageService.setSupportsBatchDelete(true); } @Test @@ -2546,15 +2590,6 @@ public void testRemoveLastNumberOfBytesManifestPartPutObjectException() throws E assertEquals(7L, info.getOffset().longValue()); } - @Test - public void testSupportsBatchDeleteConfiguration() { - assertTrue(storageService.isSupportsBatchDelete()); - storageService.setSupportsBatchDelete(false); - assertFalse(storageService.isSupportsBatchDelete()); - storageService.setSupportsBatchDelete(true); - assertTrue(storageService.isSupportsBatchDelete()); - } - @Test public void testCloseInterruptedGracefully() throws Exception { Thread.currentThread().interrupt(); @@ -2796,7 +2831,10 @@ public void testPruneOrphanPartsDeletesUnmanifestedS3Part() throws Exception { a != null && "metadata/orphan-test-id.part.".equals(a.prefix())))) .thenReturn(Arrays.asList(res1, res2)); - storageService.append(info, new ByteArrayInputStream(new byte[0])); + storageService.pruneOrphanParts( + info.getId(), + Collections.singletonList("metadata/orphan-test-id.part.00001"), + Collections.emptyList()); verify(minioClient) .removeObject( @@ -2848,6 +2886,82 @@ public void testAppendSetsLengthWhenInfoLengthIsNull() throws Exception { assertEquals(Long.valueOf(500L), result.getLength()); } + @Test + public void testAppendReusesPassedUploadInfoDirectlyWithoutRemoteGet() throws Exception { + UploadId uploadId = new UploadId("reusable-upload-id"); + UploadInfo appendUpload = new UploadInfo(); + appendUpload.setId(uploadId); + appendUpload.setStorageUploadId("storage-upload-123"); + appendUpload.setLength(1000L); + appendUpload.setOffset(0L); + + UploadInfo result = storageService.append(appendUpload, new ByteArrayInputStream(new byte[0])); + assertNotNull(result); + assertEquals(uploadId, result.getId()); + assertEquals(Long.valueOf(1000L), result.getLength()); + + // Verify remote GET metadata/.info was never invoked because caller provided populated + // UploadInfo + verify(minioClient, never()) + .getObject( + argThat( + (GetObjectArgs a) -> + a != null && a.object().equals("metadata/reusable-upload-id.info"))); + } + + @Test + public void testUploadExecutorRejectedExecutionHandler() throws Exception { + ThreadPoolExecutor executor = storageService.getUploadExecutor(); + RejectedExecutionHandler handler = executor.getRejectedExecutionHandler(); + assertNotNull(handler); + + // 1. Rejected when executor is shut down + ThreadPoolExecutor mockExecutor = mock(ThreadPoolExecutor.class); + when(mockExecutor.isShutdown()).thenReturn(true); + try { + handler.rejectedExecution(() -> {}, mockExecutor); + fail("Expected RejectedExecutionException when shut down"); + } catch (RejectedExecutionException e) { + assertTrue(e.getMessage().contains("shut down")); + } + + // 2. Enqueued when executor is active + BlockingQueue queue = new LinkedBlockingQueue<>(); + when(mockExecutor.isShutdown()).thenReturn(false); + when(mockExecutor.getQueue()).thenReturn(queue); + Runnable task = () -> {}; + handler.rejectedExecution(task, mockExecutor); + assertEquals(1, queue.size()); + + // 3. Interrupted while enqueuing + @SuppressWarnings("unchecked") + BlockingQueue fullQueue = mock(BlockingQueue.class); + doThrow(new InterruptedException("interrupted")).when(fullQueue).put(any(Runnable.class)); + when(mockExecutor.getQueue()).thenReturn(fullQueue); + try { + handler.rejectedExecution(task, mockExecutor); + fail("Expected RejectedExecutionException on interrupt"); + } catch (RejectedExecutionException e) { + assertTrue(e.getMessage().contains("Interrupted")); + assertTrue(Thread.interrupted()); // clears interrupted flag + } + } + + @Test + public void testAppendUpdatesExpirationPeriodWhenConfigured() throws Exception { + storageService.setUploadExpirationPeriod(7200L); + UploadId uploadId = new UploadId("expire-test-id"); + UploadInfo info = new UploadInfo(); + info.setId(uploadId); + info.setStorageUploadId("storage-up-1"); + info.setOffset(0L); + info.setLength(100L); + + UploadInfo result = storageService.append(info, new ByteArrayInputStream(new byte[0])); + assertNotNull(result); + assertNotNull(result.getExpirationTimestamp()); + } + @Test public void testGetUploadInfoByChecksumNoSuchKeyReturnsNull() throws Exception { storageService.setUploadDeduplicationEnabled(true); @@ -2919,7 +3033,7 @@ public void testPruneOrphanedChecksumIndicesMissingData() throws Exception { storageService.cleanupExpiredUploads(null); - verify(minioClient).removeObjects(any(RemoveObjectsArgs.class)); + verify(minioClient, atLeastOnce()).removeObjects(any(RemoveObjectsArgs.class)); verify(minioClient) .removeObject( argThat((RemoveObjectArgs a) -> a != null && a.object().equals("uploads/parent-123"))); @@ -2978,7 +3092,7 @@ public void testPruneOrphanedChecksumIndicesAbandonedTombstone() throws Exceptio storageService.cleanupExpiredUploads(null); - verify(minioClient).removeObjects(any(RemoveObjectsArgs.class)); + verify(minioClient, atLeastOnce()).removeObjects(any(RemoveObjectsArgs.class)); verify(minioClient) .removeObject( argThat( diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java index 63d5b831..690a1347 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UploadLockTest.java @@ -4,6 +4,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import io.minio.MinioClient; @@ -11,14 +12,21 @@ import io.minio.RemoveObjectArgs; import io.minio.RemoveObjectsArgs; import io.minio.Result; +import io.minio.errors.ErrorResponseException; import io.minio.messages.DeleteResult; +import io.minio.messages.ErrorResponse; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import me.desair.tus.server.upload.LeaseData; +import me.desair.tus.server.upload.UploadId; import me.desair.tus.server.util.InterruptibleInputStream; import me.desair.tus.server.util.LeaseDataJsonSerializer; import org.junit.Before; @@ -28,7 +36,8 @@ public class S3UploadLockTest { private MinioClient minioClient; - private ConcurrentMap inputStreamMap; + private ConcurrentMap inputStreamMap; + private final UploadId defaultUploadId = new UploadId("upload-1"); @Before public void setUp() { @@ -43,12 +52,13 @@ private LeaseData createLeaseData(String holderId, String requestUri) { @Test public void testLockGettersReleaseAndRenewLease() throws Exception { InputStream mockStream = mock(InputStream.class); - inputStreamMap.put("/files/upload-1", mockStream); + inputStreamMap.put(defaultUploadId, mockStream); LeaseData leaseData = createLeaseData("holder-123", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( leaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -78,6 +88,7 @@ public void testRenewLeaseExceptionHandling() throws Exception { S3UploadLock lock = new S3UploadLock( leaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -96,13 +107,15 @@ public void testLockDeleteQuietlyWithNullKeysAndExceptionHandling() throws Excep LeaseData leaseData = createLeaseData("holder-123", "/files/upload-1"); S3UploadLock lockWithNullKeys = - new S3UploadLock(leaseData, minioClient, "test-bucket", null, null, inputStreamMap); + new S3UploadLock( + leaseData, defaultUploadId, minioClient, "test-bucket", null, null, inputStreamMap); lockWithNullKeys.close(); S3UploadLock lockWithKeys = new S3UploadLock( leaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -114,15 +127,18 @@ public void testLockDeleteQuietlyWithNullKeysAndExceptionHandling() throws Excep } @Test - public void testCloseHeartbeatExecutorShutdownException() throws Exception { - java.util.concurrent.ScheduledExecutorService mockExecutor = - mock(java.util.concurrent.ScheduledExecutorService.class); - Mockito.doThrow(new RuntimeException("Shutdown error")).when(mockExecutor).shutdownNow(); + public void testCloseCancelsHeartbeatTask() throws Exception { + ScheduledExecutorService mockExecutor = mock(ScheduledExecutorService.class); + ScheduledFuture mockFuture = mock(ScheduledFuture.class); + Mockito.doReturn(mockFuture) + .when(mockExecutor) + .scheduleWithFixedDelay(any(), anyLong(), anyLong(), any()); LeaseData myLeaseData = createLeaseData("holder-123", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -131,6 +147,7 @@ public void testCloseHeartbeatExecutorShutdownException() throws Exception { mockExecutor); lock.close(); + Mockito.verify(mockFuture).cancel(true); assertEquals("holder-123", lock.getHolderId()); } @@ -153,19 +170,19 @@ public void testRenewLeaseSkipsWhenHolderMismatch() throws Exception { "test-bucket", "eu-central-1", "tus-locks/upload-1.lock", - new java.io.ByteArrayInputStream( - json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))); Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); InterruptibleInputStream stream = new InterruptibleInputStream(new ByteArrayInputStream("data".getBytes())); - inputStreamMap.put("/files/upload-1", stream); + inputStreamMap.put(defaultUploadId, stream); S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -202,8 +219,7 @@ public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { "test-bucket", "eu-central-1", "tus-locks/upload-1.lock", - new java.io.ByteArrayInputStream( - json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))); Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); @@ -211,6 +227,7 @@ public void testRenewLeaseSucceedsWhenHolderMatches() throws Exception { S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -230,6 +247,7 @@ public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -242,25 +260,68 @@ public void testDoesLockOwnershipMatchNullChecksAndException() throws Exception // Null minioClient returns false S3UploadLock nullClientLock = new S3UploadLock( - myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); + myLeaseData, + defaultUploadId, + null, + "test-bucket", + "tus-locks/upload-1.lock", + null, + inputStreamMap); org.junit.Assert.assertFalse(nullClientLock.doesLockOwnershipMatch("key")); // Null bucket returns false S3UploadLock nullBucketLock = new S3UploadLock( - myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); + myLeaseData, + defaultUploadId, + minioClient, + null, + "tus-locks/upload-1.lock", + null, + inputStreamMap); org.junit.Assert.assertFalse(nullBucketLock.doesLockOwnershipMatch("key")); - // General exception (non-ErrorResponseException) returns true to allow proceed + // General exception returns false (fail-closed split-brain protection) Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))) .thenThrow(new RuntimeException("Transient S3 error")); + org.junit.Assert.assertFalse(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); + + // ErrorResponseException with NoSuchKey returns false + ErrorResponse noSuchKeyResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(noSuchKeyResponse.code()).thenReturn("NoSuchKey"); + ErrorResponseException notFoundEx = new ErrorResponseException(noSuchKeyResponse, null, null); + Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenThrow(notFoundEx); + org.junit.Assert.assertFalse(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); + + // ErrorResponseException with non-NoSuchKey (e.g. AccessDenied) returns false + ErrorResponse accessDeniedResponse = Mockito.mock(ErrorResponse.class); + Mockito.when(accessDeniedResponse.code()).thenReturn("AccessDenied"); + ErrorResponseException accessDeniedEx = + new ErrorResponseException(accessDeniedResponse, null, null); + Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))) + .thenThrow(accessDeniedEx); + org.junit.Assert.assertFalse(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); + + // Matching holder returns true + String matchJson = LeaseDataJsonSerializer.serialize(myLeaseData); + io.minio.GetObjectResponse matchResponse = + new io.minio.GetObjectResponse( + null, + "test-bucket", + "eu-central-1", + "tus-locks/upload-1.lock", + new ByteArrayInputStream(matchJson.getBytes(StandardCharsets.UTF_8))); + Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))) + .thenReturn(matchResponse); org.junit.Assert.assertTrue(lock.doesLockOwnershipMatch("tus-locks/upload-1.lock")); - // Renew lease with null lockKey or null minioClient + // Renew lease and release resource with null lockKey or null minioClient nullClientLock.doRenewLease(); S3UploadLock nullKeyLock = - new S3UploadLock(myLeaseData, minioClient, "test-bucket", null, null, inputStreamMap); + new S3UploadLock( + myLeaseData, defaultUploadId, minioClient, "test-bucket", null, null, inputStreamMap); nullKeyLock.doRenewLease(); + nullKeyLock.close(); } @Test @@ -283,14 +344,17 @@ public void testReleaseLockResourceWhenOwnershipMismatch() throws Exception { "test-bucket", "eu-central-1", "tus-locks/upload-1.lock", - new ByteArrayInputStream(json.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8))); Mockito.when(minioClient.getObject(any(io.minio.GetObjectArgs.class))).thenReturn(response); LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + // Expire local lease so releaseLockResource triggers remote ownership verification + myLeaseData.setExpiresAt(System.currentTimeMillis() - 5000L); S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -305,12 +369,34 @@ public void testReleaseLockResourceWhenOwnershipMismatch() throws Exception { Mockito.verify(minioClient, Mockito.never()).removeObject(any(RemoveObjectArgs.class)); } + @Test + public void testReleaseLockResourceFastPathWhenUnexpired() throws Exception { + LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); + // Lease is unexpired (60s in future): releaseLockResource must skip remote GET verification + S3UploadLock lock = + new S3UploadLock( + myLeaseData, + defaultUploadId, + minioClient, + "test-bucket", + "tus-locks/upload-1.lock", + "tus-locks/upload-1.stop", + inputStreamMap); + + lock.close(); + + // Remote getObject must NEVER be invoked on fast-path release + Mockito.verify(minioClient, Mockito.never()).getObject(any(io.minio.GetObjectArgs.class)); + Mockito.verify(minioClient).removeObjects(any(RemoveObjectsArgs.class)); + } + @Test public void testDeleteS3ObjectsNullChecksAndEmpty() { LeaseData myLeaseData = createLeaseData("my-holder", "/files/upload-1"); S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -320,13 +406,25 @@ public void testDeleteS3ObjectsNullChecksAndEmpty() { // Null client S3UploadLock nullClientLock = new S3UploadLock( - myLeaseData, null, "test-bucket", "tus-locks/upload-1.lock", null, inputStreamMap); + myLeaseData, + defaultUploadId, + null, + "test-bucket", + "tus-locks/upload-1.lock", + null, + inputStreamMap); nullClientLock.deleteS3Objects("key1", "key2"); // Null bucket S3UploadLock nullBucketLock = new S3UploadLock( - myLeaseData, minioClient, null, "tus-locks/upload-1.lock", null, inputStreamMap); + myLeaseData, + defaultUploadId, + minioClient, + null, + "tus-locks/upload-1.lock", + null, + inputStreamMap); nullBucketLock.deleteS3Objects("key1", "key2"); // Both keys null -> empty objects @@ -344,6 +442,7 @@ public void testDeleteS3ObjectsWithErrorsAndExceptions() throws Exception { S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -380,6 +479,7 @@ public void testDeleteS3ObjectsWhenMinioClientThrows() { S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -400,6 +500,7 @@ public void testDeleteSingleObjectEdgeCases() throws Exception { S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -409,11 +510,13 @@ public void testDeleteSingleObjectEdgeCases() throws Exception { lock.deleteSingleObject(null); S3UploadLock nullClientLock = - new S3UploadLock(myLeaseData, null, "test-bucket", "k1", "k2", inputStreamMap); + new S3UploadLock( + myLeaseData, defaultUploadId, null, "test-bucket", "k1", "k2", inputStreamMap); nullClientLock.deleteSingleObject("k1"); S3UploadLock nullBucketLock = - new S3UploadLock(myLeaseData, minioClient, null, "k1", "k2", inputStreamMap); + new S3UploadLock( + myLeaseData, defaultUploadId, minioClient, null, "k1", "k2", inputStreamMap); nullBucketLock.deleteSingleObject("k1"); Mockito.doThrow(new RuntimeException("removeObject error")) @@ -429,6 +532,7 @@ public void testDeleteS3ObjectsResultIteratorExceptionFallback() throws Exceptio S3UploadLock lock = new S3UploadLock( myLeaseData, + defaultUploadId, minioClient, "test-bucket", "tus-locks/upload-1.lock", @@ -436,12 +540,12 @@ public void testDeleteS3ObjectsResultIteratorExceptionFallback() throws Exceptio inputStreamMap); @SuppressWarnings("unchecked") - io.minio.Result mockResult = mock(io.minio.Result.class); + Result mockResult = mock(Result.class); Mockito.when(mockResult.get()) .thenThrow(new RuntimeException("Simulated error while reading delete result")); Mockito.when(minioClient.removeObjects(any(RemoveObjectsArgs.class))) - .thenReturn(java.util.Collections.singletonList(mockResult)); + .thenReturn(Collections.singletonList(mockResult)); lock.deleteS3Objects("tus-locks/upload-1.lock", "tus-locks/upload-1.stop"); diff --git a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java index 5dc3b3c9..5db265d8 100644 --- a/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java +++ b/src/test/java/me/desair/tus/server/upload/s3/S3UtilsTest.java @@ -1,13 +1,254 @@ package me.desair.tus.server.upload.s3; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import io.minio.GetObjectArgs; +import io.minio.GetObjectResponse; +import io.minio.MinioClient; +import io.minio.RemoveObjectArgs; +import io.minio.RemoveObjectsArgs; +import io.minio.Result; +import io.minio.StatObjectArgs; +import io.minio.StatObjectResponse; import io.minio.errors.ErrorResponseException; +import io.minio.messages.DeleteResult; import io.minio.messages.ErrorResponse; +import io.minio.messages.Item; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; import org.junit.Test; public class S3UtilsTest { + @Test + public void testBuildMinioClient() { + MinioClient clientWithRegion = + S3Utils.buildMinioClient("https://s3.amazonaws.com", "eu-central-1", "access", "secret"); + assertNotNull(clientWithRegion); + + MinioClient clientDefaultRegion = + S3Utils.buildMinioClient("https://s3.amazonaws.com", null, "access", "secret"); + assertNotNull(clientDefaultRegion); + + MinioClient clientEmptyRegion = + S3Utils.buildMinioClient("https://s3.amazonaws.com", "", "access", "secret"); + assertNotNull(clientEmptyRegion); + } + + @Test + public void testSanitizePrefix() { + assertEquals("", S3Utils.sanitizePrefix(null)); + assertEquals("", S3Utils.sanitizePrefix("")); + assertEquals("", S3Utils.sanitizePrefix(" ")); + assertEquals("", S3Utils.sanitizePrefix("///")); + assertEquals("locks/", S3Utils.sanitizePrefix("locks")); + assertEquals("locks/", S3Utils.sanitizePrefix("/locks")); + assertEquals("locks/", S3Utils.sanitizePrefix("locks/")); + assertEquals("locks/", S3Utils.sanitizePrefix("/locks/")); + assertEquals("a/b/c/", S3Utils.sanitizePrefix("/a/b/c")); + assertEquals("custom/", S3Utils.sanitizePrefix("/custom///")); + } + + @Test + public void testDeleteS3Object() throws Exception { + MinioClient client = mock(MinioClient.class); + + // Null checks (should not throw) + S3Utils.deleteS3Object(null, "bucket", "key"); + S3Utils.deleteS3Object(client, null, "key"); + S3Utils.deleteS3Object(client, "bucket", null); + + // Normal deletion + S3Utils.deleteS3Object(client, "test-bucket", "test-key"); + verify(client, times(1)).removeObject(any(RemoveObjectArgs.class)); + + // Exception propagated + doThrow(new RuntimeException("Network error")) + .when(client) + .removeObject(any(RemoveObjectArgs.class)); + try { + S3Utils.deleteS3Object(client, "test-bucket", "test-key"); + org.junit.Assert.fail("Expected RuntimeException to be thrown"); + } catch (RuntimeException e) { + assertEquals("Network error", e.getMessage()); + } + } + + @Test + public void testDeleteS3ObjectQuietly() throws Exception { + MinioClient client = mock(MinioClient.class); + + // Null checks (should not throw) + S3Utils.deleteS3ObjectQuietly(null, "bucket", "key"); + S3Utils.deleteS3ObjectQuietly(client, null, "key"); + S3Utils.deleteS3ObjectQuietly(client, "bucket", null); + + // Normal deletion + S3Utils.deleteS3ObjectQuietly(client, "test-bucket", "test-key"); + verify(client, times(1)).removeObject(any(RemoveObjectArgs.class)); + + // Exception handled quietly + doThrow(new RuntimeException("Network error")) + .when(client) + .removeObject(any(RemoveObjectArgs.class)); + S3Utils.deleteS3ObjectQuietly(client, "test-bucket", "test-key"); + } + + @Test + public void testGetObjectInputStream() throws Exception { + MinioClient client = mock(MinioClient.class); + + // Null checks + assertNull(S3Utils.getObjectInputStream(null, "bucket", "key")); + assertNull(S3Utils.getObjectInputStream(client, null, "key")); + assertNull(S3Utils.getObjectInputStream(client, "bucket", null)); + + // Successful stream retrieval + GetObjectResponse mockResponse = mock(GetObjectResponse.class); + when(client.getObject(any(GetObjectArgs.class))).thenReturn(mockResponse); + + InputStream resultStream = S3Utils.getObjectInputStream(client, "bucket", "key"); + assertNotNull(resultStream); + assertEquals(mockResponse, resultStream); + + // Exception propagation + when(client.getObject(any(GetObjectArgs.class))) + .thenThrow(new RuntimeException("S3 connection dropped")); + try { + S3Utils.getObjectInputStream(client, "bucket", "key"); + org.junit.Assert.fail("Expected RuntimeException to be thrown"); + } catch (RuntimeException e) { + assertEquals("S3 connection dropped", e.getMessage()); + } + } + + @Test + public void testDeleteS3ObjectsQuietly() throws Exception { + MinioClient client = mock(MinioClient.class); + + // Null and empty checks (should not throw) + S3Utils.deleteS3ObjectsQuietly(null, "bucket", Collections.singletonList("key")); + S3Utils.deleteS3ObjectsQuietly(client, null, Collections.singletonList("key")); + S3Utils.deleteS3ObjectsQuietly(client, "bucket", null); + S3Utils.deleteS3ObjectsQuietly(client, "bucket", Collections.emptyList()); + S3Utils.deleteS3ObjectsQuietly(client, "bucket", Collections.singletonList(null)); + S3Utils.deleteS3ObjectsQuietly(client, "bucket", Arrays.asList(" ", " ", "")); + + // 1. Successful batch deletion (no errors returned by S3) + when(client.removeObjects(any(RemoveObjectsArgs.class))).thenReturn(Collections.emptyList()); + S3Utils.deleteS3ObjectsQuietly(client, "test-bucket", Arrays.asList("key1", "key2")); + verify(client, times(1)).removeObjects(any(RemoveObjectsArgs.class)); + verify(client, times(0)).removeObject(any(RemoveObjectArgs.class)); + + // 2. Batch deletion reporting error item (falls back to individual deletion) + DeleteResult.Error mockError = mock(DeleteResult.Error.class); + when(mockError.objectName()).thenReturn("key1"); + when(mockError.message()).thenReturn("AccessDenied"); + Result result = new Result<>(mockError); + List> errorResults = Collections.singletonList(result); + when(client.removeObjects(any(RemoveObjectsArgs.class))).thenReturn(errorResults); + + S3Utils.deleteS3ObjectsQuietly(client, "test-bucket", Arrays.asList("key1", "key2")); + verify(client, times(2)).removeObjects(any(RemoveObjectsArgs.class)); + verify(client, times(2)).removeObject(any(RemoveObjectArgs.class)); + + // 3. Batch failure throwing exception (falls back to individual deletion) + doThrow(new RuntimeException("Batch delete unsupported")) + .when(client) + .removeObjects(any(RemoveObjectsArgs.class)); + S3Utils.deleteS3ObjectsQuietly(client, "test-bucket", Arrays.asList("key3", "key4")); + verify(client, times(3)).removeObjects(any(RemoveObjectsArgs.class)); + verify(client, times(4)).removeObject(any(RemoveObjectArgs.class)); + } + + @Test + public void testExtractObjectName() throws Exception { + assertNull(S3Utils.extractObjectName(null)); + + // Result containing null item + @SuppressWarnings("unchecked") + Result nullResult = mock(Result.class); + when(nullResult.get()).thenReturn(null); + assertNull(S3Utils.extractObjectName(nullResult)); + + // Result containing valid item + Item item = mock(Item.class); + when(item.objectName()).thenReturn("part.00001"); + @SuppressWarnings("unchecked") + Result validResult = mock(Result.class); + when(validResult.get()).thenReturn(item); + assertEquals("part.00001", S3Utils.extractObjectName(validResult)); + + // Result throwing exception + @SuppressWarnings("unchecked") + Result errorResult = mock(Result.class); + when(errorResult.get()).thenThrow(new RuntimeException("Failed reading item")); + try { + S3Utils.extractObjectName(errorResult); + org.junit.Assert.fail("Expected RuntimeException to be thrown"); + } catch (RuntimeException e) { + assertEquals("Failed to read S3 item result", e.getMessage()); + } + } + + @Test + public void testGetObjectSize() throws Exception { + MinioClient client = mock(MinioClient.class); + + // Null checks + assertEquals(-1L, S3Utils.getObjectSize(null, "bucket", "key")); + assertEquals(-1L, S3Utils.getObjectSize(client, null, "key")); + assertEquals(-1L, S3Utils.getObjectSize(client, "bucket", null)); + + // stat returns null + when(client.statObject(any(StatObjectArgs.class))).thenReturn(null); + assertEquals(-1L, S3Utils.getObjectSize(client, "bucket", "key")); + + // stat returns response with size + StatObjectResponse statResponse = mock(StatObjectResponse.class); + when(statResponse.size()).thenReturn(1024L); + when(client.statObject(any(StatObjectArgs.class))).thenReturn(statResponse); + assertEquals(1024L, S3Utils.getObjectSize(client, "bucket", "key")); + + // stat throws exception + when(client.statObject(any(StatObjectArgs.class))).thenThrow(new RuntimeException("NoSuchKey")); + assertEquals(-1L, S3Utils.getObjectSize(client, "bucket", "key")); + } + + @Test + public void testObjectExists() throws Exception { + MinioClient client = mock(MinioClient.class); + + // Null checks + assertFalse(S3Utils.objectExists(null, "bucket", "key")); + assertFalse(S3Utils.objectExists(client, null, "key")); + assertFalse(S3Utils.objectExists(client, "bucket", null)); + + // Object exists (statObject returns response) + StatObjectResponse statResponse = mock(StatObjectResponse.class); + when(client.statObject(any(StatObjectArgs.class))).thenReturn(statResponse); + assertTrue(S3Utils.objectExists(client, "bucket", "key")); + + // Object does not exist (statObject throws exception) + when(client.statObject(any(StatObjectArgs.class))).thenThrow(new RuntimeException("NoSuchKey")); + assertFalse(S3Utils.objectExists(client, "bucket", "key")); + } + @Test public void testParseErrorResponseNull() { assertEquals(S3ErrorType.UNKNOWN, S3Utils.parseErrorResponse(null)); @@ -44,22 +285,22 @@ public void testParseErrorResponseCodes() throws Exception { @Test public void testParseErrorResponseHttp501() { - okhttp3.Response httpResponse = - new okhttp3.Response.Builder() - .request(new okhttp3.Request.Builder().url("https://example.com").build()) - .protocol(okhttp3.Protocol.HTTP_1_1) + Response httpResponse = + new Response.Builder() + .request(new Request.Builder().url("https://example.com").build()) + .protocol(Protocol.HTTP_1_1) .code(501) .message("Not Implemented") .build(); - ErrorResponse errorResponse = org.mockito.Mockito.mock(ErrorResponse.class); - org.mockito.Mockito.when(errorResponse.code()).thenReturn(""); + ErrorResponse errorResponse = mock(ErrorResponse.class); + when(errorResponse.code()).thenReturn(""); ErrorResponseException ex = new ErrorResponseException(errorResponse, httpResponse, null); assertEquals(S3ErrorType.API_NOT_IMPLEMENTED, S3Utils.parseErrorResponse(ex)); } private ErrorResponseException createExceptionWithCode(String code) { - ErrorResponse errorResponse = org.mockito.Mockito.mock(ErrorResponse.class); - org.mockito.Mockito.when(errorResponse.code()).thenReturn(code); + ErrorResponse errorResponse = mock(ErrorResponse.class); + when(errorResponse.code()).thenReturn(code); return new ErrorResponseException(errorResponse, null, null); } } diff --git a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java index 60f9b921..c765cbae 100644 --- a/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java +++ b/src/test/java/me/desair/tus/server/upload/util/AsyncChunkUploaderTest.java @@ -4,6 +4,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.ByteArrayInputStream; import java.io.File; @@ -12,9 +15,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -44,9 +49,10 @@ public void setUp() throws Exception { public void tearDown() throws Exception { if (executor != null) { executor.shutdownNow(); + executor.awaitTermination(2, TimeUnit.SECONDS); } if (tempDir != null && tempDir.exists()) { - org.apache.commons.io.FileUtils.deleteDirectory(tempDir); + org.apache.commons.io.FileUtils.deleteQuietly(tempDir); } } @@ -748,7 +754,17 @@ public void testAbortWithActiveInFlightUploadWaitsAndCancels() throws Exception uploader.abort(); hangLatch.countDown(); - assertFalse("File should be cleaned up by abort", f1.exists()); + // Allow up to 3 seconds for asynchronous background worker thread to execute its finally + // cleanup + boolean deleted = false; + for (int i = 0; i < 60; i++) { + if (!f1.exists()) { + deleted = true; + break; + } + Thread.sleep(50); + } + assertTrue("File should be cleaned up by worker thread finally block", deleted); } @Test @@ -782,6 +798,27 @@ public void testWaitForInFlightUploadTimeoutPolling() throws Exception { } } + @Test + public void testSubmitChunkThrowsDeletesTempFileImmediately() throws Exception { + ExecutorService mockExecutor = mock(ExecutorService.class); + @SuppressWarnings("unchecked") + java.util.concurrent.Future future = mock(java.util.concurrent.Future.class); + when(mockExecutor.submit(any(Callable.class))) + .thenThrow(new RejectedExecutionException("Executor saturated")); + + File f = new File(tempDir, "reject-f.tmp"); + Files.write(f.toPath(), new byte[] {1, 2, 3}); + assertTrue(f.exists()); + + try (AsyncChunkUploader uploader = new AsyncChunkUploader(mockExecutor)) { + uploader.submitChunk(f, 3, "k-rej", () -> {}); + fail("Expected RejectedExecutionException"); + } catch (RejectedExecutionException e) { + // Temp file must be immediately deleted upon submit failure + assertFalse(f.exists()); + } + } + @Test public void testCheckAndConfirmInFlightInterruptedThrowsIOException() throws Exception { File f1 = new File(tempDir, "interrupt-f1.tmp"); diff --git a/src/test/java/me/desair/tus/server/util/UtilsTest.java b/src/test/java/me/desair/tus/server/util/UtilsTest.java index 5e9c0aaf..50a639d5 100644 --- a/src/test/java/me/desair/tus/server/util/UtilsTest.java +++ b/src/test/java/me/desair/tus/server/util/UtilsTest.java @@ -201,22 +201,6 @@ public void testWriteSerializableNullPath() throws Exception { Utils.writeSerializable(null, Paths.get("target", "ignored.bin")); } - @Test - public void testCreateTempSiblingPath() { - assertThat(Utils.createTempSiblingPath(null), is(nullValue())); - - Path absPath = Paths.get("target", "test-dir", "data.json").toAbsolutePath(); - Path tempAbs = Utils.createTempSiblingPath(absPath); - assertThat(tempAbs, is(notNullValue())); - assertThat(tempAbs.getParent(), is(absPath.getParent())); - assertThat(tempAbs.getFileName().toString().startsWith("data.json.tmp."), is(true)); - - Path relPathNoParent = Paths.get("data.json"); - Path tempRelNoParent = Utils.createTempSiblingPath(relPathNoParent); - assertThat(tempRelNoParent, is(notNullValue())); - assertThat(tempRelNoParent.getFileName().toString().startsWith("data.json.tmp."), is(true)); - } - @Test public void testDeletePathQuietly() throws Exception { assertThat(Utils.deletePathQuietly(null), is(false)); @@ -227,48 +211,6 @@ public void testDeletePathQuietly() throws Exception { assertThat(Files.exists(tempFile), is(false)); } - @Test - public void testTempPathAutoCloseable() throws Exception { - Path targetFile = Paths.get("target", "temp-target-" + UUID.randomUUID() + ".json"); - Path tempFilePath; - - try (Utils.TempPath tempPath = Utils.createTempSibling(targetFile)) { - tempFilePath = tempPath.getPath(); - assertThat(tempFilePath, is(notNullValue())); - Files.write(tempFilePath, "temp data".getBytes()); - assertThat(Files.exists(tempFilePath), is(true)); - } - - // AutoCloseable should have automatically deleted the temp file - assertThat(Files.exists(tempFilePath), is(false)); - - // Null path handling - try (Utils.TempPath nullTempPath = new Utils.TempPath(null)) { - assertThat(nullTempPath.getPath(), is(nullValue())); - } - } - - @Test - public void testAtomicMove() throws Exception { - // Null inputs should not throw exception - Utils.atomicMove(null, Paths.get("dest")); - Utils.atomicMove(Paths.get("src"), null); - - Path src = Files.createTempFile("tus-atomic-src", ".tmp"); - Path dst = Files.createTempFile("tus-atomic-dst", ".tmp"); - try { - Files.write(src, "atomic test content".getBytes()); - Utils.atomicMove(src, dst); - - assertThat(Files.exists(src), is(false)); - assertThat(Files.exists(dst), is(true)); - assertThat(new String(Files.readAllBytes(dst)), is("atomic test content")); - } finally { - Files.deleteIfExists(src); - Files.deleteIfExists(dst); - } - } - @Test public void testLockFileExclusivelyAndShared() throws Exception { Path tempFile = Files.createTempFile("tus-test-lock", ".tmp"); @@ -831,44 +773,6 @@ public void interrupt() { Utils.interruptStream(faultyStream); } - @Test - public void testInterruptThreadNull() { - // Should do nothing without exception - Utils.interruptThread(null); - } - - @Test - public void testInterruptThreadAlive() throws Exception { - java.util.concurrent.atomic.AtomicBoolean wasInterrupted = - new java.util.concurrent.atomic.AtomicBoolean(false); - Thread thread = - new Thread( - () -> { - try { - Thread.sleep(5000L); - } catch (InterruptedException e) { - wasInterrupted.set(true); - } - }, - "test-interruptible-thread"); - thread.start(); - - // Give the thread a moment to start - Thread.sleep(50L); - - Utils.interruptThread(thread); - thread.join(2000L); - - assertThat(wasInterrupted.get(), is(true)); - } - - @Test - public void testInterruptThreadDead() { - Thread unstartedThread = new Thread(() -> {}, "test-unstarted"); - // Should handle cleanly without exception - Utils.interruptThread(unstartedThread); - } - @Test public void ensureDirectoryExistsWithNullPathShouldDoNothing() throws Exception { Utils.ensureDirectoryExists(null);