From 0cb54f5390cfdc975928f083e5f6f46761c24e6b Mon Sep 17 00:00:00 2001 From: scarreras Date: Tue, 13 Oct 2026 15:31:22 -0300 Subject: [PATCH 1/2] Add sixteen sdkharness resilience checks: retry, backoff, reauthorization Adds the resilience level to the .sdkharness/ contract: one scenario per run, named by SDKHARNESS_SCENARIO, run by tests/run-resilience. Each check is a Java program that arms one documented fault through the simulator's control listener, makes one SDK call with the SDK's own default retry policy, and asserts the end state (bytes read back, listing, or the error) and the recovery path from the simulator's request journal: a fresh upload URL after a 503, 500, 408, 401, reset or stall, no retry after a 403 cap_exceeded, reauthorization after a 401 and after the token's 24 h life, and the lower bound of a Retry-After or the first backoff step. The stall check sets a 2 s socket timeout through the documented HttpClientFactoryImpl.Builder.setSocketTimeoutSeconds. A check still running after 60 s is killed and reported as a failure. The harness builds the core and httpclient jars and passes them as SDKHARNESS_JAVA_CLASSPATH; the build is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .sdkharness/README.md | 51 +- .sdkharness/tests.tsv | 16 + .../tests/resilience/ApiBackoff503.java | 45 ++ .../tests/resilience/ApiRetryAfter429.java | 45 ++ .../tests/resilience/ApiRetryAfter503.java | 45 ++ .../tests/resilience/AuthClockExpiry.java | 47 ++ .../tests/resilience/AuthExpired401.java | 43 ++ .../tests/resilience/DownloadRetry503.java | 39 ++ .../tests/resilience/PartRetry503.java | 54 +++ .sdkharness/tests/resilience/Support.java | 437 ++++++++++++++++++ .../resilience/UploadCapExceeded403.java | 47 ++ .../resilience/UploadExpiredToken401.java | 36 ++ .../tests/resilience/UploadGetUrl503.java | 45 ++ .../resilience/UploadResetBeforeResponse.java | 39 ++ .../resilience/UploadResetMidRequest.java | 34 ++ .../tests/resilience/UploadRetry408.java | 34 ++ .../tests/resilience/UploadRetry500.java | 34 ++ .../tests/resilience/UploadRetry503.java | 36 ++ .sdkharness/tests/resilience/UploadStall.java | 36 ++ .sdkharness/tests/run-resilience | 101 ++++ 20 files changed, 1259 insertions(+), 5 deletions(-) create mode 100644 .sdkharness/tests/resilience/ApiBackoff503.java create mode 100644 .sdkharness/tests/resilience/ApiRetryAfter429.java create mode 100644 .sdkharness/tests/resilience/ApiRetryAfter503.java create mode 100644 .sdkharness/tests/resilience/AuthClockExpiry.java create mode 100644 .sdkharness/tests/resilience/AuthExpired401.java create mode 100644 .sdkharness/tests/resilience/DownloadRetry503.java create mode 100644 .sdkharness/tests/resilience/PartRetry503.java create mode 100644 .sdkharness/tests/resilience/Support.java create mode 100644 .sdkharness/tests/resilience/UploadCapExceeded403.java create mode 100644 .sdkharness/tests/resilience/UploadExpiredToken401.java create mode 100644 .sdkharness/tests/resilience/UploadGetUrl503.java create mode 100644 .sdkharness/tests/resilience/UploadResetBeforeResponse.java create mode 100644 .sdkharness/tests/resilience/UploadResetMidRequest.java create mode 100644 .sdkharness/tests/resilience/UploadRetry408.java create mode 100644 .sdkharness/tests/resilience/UploadRetry500.java create mode 100644 .sdkharness/tests/resilience/UploadRetry503.java create mode 100644 .sdkharness/tests/resilience/UploadStall.java create mode 100755 .sdkharness/tests/run-resilience diff --git a/.sdkharness/README.md b/.sdkharness/README.md index 3c4d0bac1..4e9210878 100644 --- a/.sdkharness/README.md +++ b/.sdkharness/README.md @@ -5,7 +5,7 @@ assertions in `tests/`; the harness builds this checkout, starts a fresh local B2 simulator, and retains and reports the evidence. -Scenarios so far: `health/golden-path` and eight `conformance` scenarios (below). `tests/run-health` runs +Scenarios so far: `health/golden-path` and eight `conformance` and sixteen `resilience` scenarios (below). `tests/run-health` runs `tests/health/GoldenPath.java` as a single-file Java program that authorizes, uploads a small file, downloads it and compares the bytes, lists it, deletes it and confirms it is gone, then prints one result line: @@ -41,18 +41,53 @@ formatted to one capped printable line from exception class names, status codes assertions; that is formatting, not redaction, and other lines the check prints are forwarded as they are. The only credential in play is the simulator's fixed test one. +## Resilience + +`tests/run-resilience` runs one scenario per call, named by `SDKHARNESS_SCENARIO`; every +`resilience` row in `tests.tsv` points at it. Each scenario is a Java program in +`tests/resilience/` that arms **one** documented fault through the simulator's control listener +(`SDKHARNESS_SIMULATOR_CONTROL_URL`), makes one SDK call with the SDK's own default retry policy, +and asserts two things: the end state (the bytes read back intact, the listing succeeds, or the +error reaches the caller) and the recovery path, read from the simulator's request journal +(`GET /journal`: endpoint, status, fault, an opaque upload URL id; never a header, token or body). +Each file cites the B2 pages it asserts. They share `tests/resilience/Support.java`; as for +conformance, the wrapper compiles `Support.java` and the check with `javac` against the +harness-built jars, then runs the check class. Each check creates its own `sdkharness-res-*` +bucket, clears every fault it armed, and deletes everything it made, on failure too. + +| Scenario | Fault, and what it asserts | +|---|---| +| `upload.retry_503`, `upload.retry_500`, `upload.retry_408` | status on `b2_upload_file`: a fresh `b2_get_upload_url`, the retried upload on a different upload URL id, one stored version, bytes intact | +| `upload.expired_token_401` | 401 `expired_auth_token` on `b2_upload_file`: a fresh upload URL, then the file lands intact | +| `upload.get_url_503` | 503 on `b2_get_upload_url`, armed before the first upload in the bucket (the SDK pools upload URLs, so after any earlier upload the fault would never fire): retried, then the file lands | +| `upload.cap_exceeded_403` | 403 `cap_exceeded`: the error reaches the caller, exactly one `b2_upload_file`, nothing stored | +| `upload.reset_mid_request`, `upload.reset_before_response` | connection reset: a fresh upload URL, the file reads back intact (a second version is accepted after a reset before the response) | +| `upload.stall` | the server holds the connection for 60 s: the check sets a 2 s socket timeout (`HttpClientFactoryImpl.Builder.setSocketTimeoutSeconds`, the default is 20 s), the stalled request is abandoned and retried on a new upload URL | +| `part.retry_503` | 503 on `b2_upload_part`: a successful part on a different URL id, `b2_finish_large_file`, bytes intact | +| `auth.expired_401` | 401 `expired_auth_token` on `b2_list_file_names`: `b2_authorize_account` again, then the listing succeeds | +| `auth.clock_expiry` | the simulator's clock moves 24 h and 1 ms: the real 401, a new authorization, the listing succeeds, all on one long-lived client | +| `api.retry_after_429`, `api.retry_after_503` | `Retry-After: 2`: one retry, the whole call took at least 2 s | +| `api.backoff_503` | no `Retry-After`: one retry, the whole call took at least 1 s | +| `download.retry_503` | 503 on `b2_download_file_by_id`: retried, the bytes are intact | + +Timing is a lower bound only, never an upper bound (a busy runner would make that flaky). A check +that is still running after 60 s is killed and reported as a failure: the slowest scenario takes +about 3 s, so a minute means the SDK never gave up on something. + +A failing scenario prints one `SDKHARNESS_RESULT ... FAIL` line with the step and the reason. + ## What the check can and cannot reach -- **Only the loopback simulator.** `run-health` and `run-conformance` refuse any - `SDKHARNESS_SIMULATOR_URL` that is not `http://127.0.0.1:`. In conformance the SDK is also +- **Only the loopback simulator.** `run-health`, `run-conformance` and `run-resilience` refuse any + `SDKHARNESS_SIMULATOR_URL` (and, for resilience, `SDKHARNESS_SIMULATOR_CONTROL_URL`) that is not `http://127.0.0.1:`. In conformance the SDK is also held to it at request time: every check builds its client from one guarded HTTP client factory (`Support.guardedFactory`, plain HTTP only) that refuses any request whose target host is not `127.0.0.1`, redirect hops included, and a refused request fails the verdict (`loopback guard -- blocked a request to non-loopback host ...`). `health` is guarded by its origin check only. `conformance/GuardSelfTest.java` proves the guard (see below). - **Only the simulator's fixed test credential** (`test-key-id` / `test-key`, in the source). - Every ambient `B2_*` variable except `B2_BUCKET_NAME` is dropped first (conformance drops all of - them and makes its own buckets). + Every ambient `B2_*` variable except `B2_BUCKET_NAME` is dropped first (conformance and resilience drop all of + them and make their own buckets). - **No build.** The harness builds the `core` and `httpclient` jars from this checkout and passes them, with their runtime dependencies, as `SDKHARNESS_JAVA_CLASSPATH`. The build files here are not changed. A missing `java` on `PATH` reports `SKIP`, which the harness @@ -70,6 +105,12 @@ javac --release 11 -d /tmp/guard -cp "$CP" .sdkharness/tests/conformance/Support java -cp "/tmp/guard:$CP" GuardSelfTest # GUARD-SELFTEST b2-sdk-java: PASS ``` +For resilience, start the simulator with its control listener (`serve.mjs --control`), set +`SDKHARNESS_TEST_LEVEL=resilience`, `SDKHARNESS_SCENARIO=`, +`SDKHARNESS_SIMULATOR_URL`, `SDKHARNESS_SIMULATOR_CONTROL_URL` and `SDKHARNESS_JAVA_CLASSPATH` +(as below), and run `.sdkharness/tests/run-resilience` (needs `javac`, and a fresh simulator per +scenario: a fault is armed once and a stalled connection is held for a minute). + Requires JDK 11 or later and a simulator (`backblaze-labs/b2-simulator`) with a bucket named `sdkharness-healthcheck`: diff --git a/.sdkharness/tests.tsv b/.sdkharness/tests.tsv index e4dd1139e..5dc227f1b 100644 --- a/.sdkharness/tests.tsv +++ b/.sdkharness/tests.tsv @@ -8,3 +8,19 @@ conformance files.delete_version simulator ./.sdkharness/tests/run-conformance conformance files.hide simulator ./.sdkharness/tests/run-conformance conformance bucket.crud simulator ./.sdkharness/tests/run-conformance conformance files.metadata simulator ./.sdkharness/tests/run-conformance +resilience upload.retry_503 simulator ./.sdkharness/tests/run-resilience +resilience upload.retry_500 simulator ./.sdkharness/tests/run-resilience +resilience upload.retry_408 simulator ./.sdkharness/tests/run-resilience +resilience upload.expired_token_401 simulator ./.sdkharness/tests/run-resilience +resilience upload.get_url_503 simulator ./.sdkharness/tests/run-resilience +resilience upload.cap_exceeded_403 simulator ./.sdkharness/tests/run-resilience +resilience upload.reset_mid_request simulator ./.sdkharness/tests/run-resilience +resilience upload.reset_before_response simulator ./.sdkharness/tests/run-resilience +resilience upload.stall simulator ./.sdkharness/tests/run-resilience +resilience part.retry_503 simulator ./.sdkharness/tests/run-resilience +resilience auth.expired_401 simulator ./.sdkharness/tests/run-resilience +resilience auth.clock_expiry simulator ./.sdkharness/tests/run-resilience +resilience api.retry_after_429 simulator ./.sdkharness/tests/run-resilience +resilience api.retry_after_503 simulator ./.sdkharness/tests/run-resilience +resilience api.backoff_503 simulator ./.sdkharness/tests/run-resilience +resilience download.retry_503 simulator ./.sdkharness/tests/run-resilience diff --git a/.sdkharness/tests/resilience/ApiBackoff503.java b/.sdkharness/tests/resilience/ApiBackoff503.java new file mode 100644 index 000000000..983f75095 --- /dev/null +++ b/.sdkharness/tests/resilience/ApiBackoff503.java @@ -0,0 +1,45 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import java.util.ArrayList; +import java.util.List; + +/** + * api.backoff_503. Run by ../run-resilience (see ../../README.md). + * + *

A 503 with no Retry-After is retried after the documented first backoff step, 1 second (a lower bound + * on the whole call), and the listing then succeeds. + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class ApiBackoff503 { + public static void main(String[] args) { + Support.run("api.backoff_503", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/api-backoff-503.bin"; + Support.upload(name, Support.payload(1024, 13)); + + Support.step("arm the fault"); + Support.armFault("b2_list_file_names", 503, "service_unavailable", null); + + Support.step("list"); + Support.markJournal(); + final List names = new ArrayList<>(); + final long elapsedMs = Support.timed(() -> names.addAll(Support.listNames())); + + Support.step("recovery path"); + final List window = Support.only(Support.sinceMark(), "b2_list_file_names"); + Support.check(window.size() >= 2 && window.get(0).status == 503 && window.get(0).fault.equals("injected"), + "the first b2_list_file_names did not carry the 503 fault: " + Support.show(window)); + Support.check(window.get(window.size() - 1).status == 200, "the listing was never retried to a 200: " + Support.show(window)); + + Support.step("backoff"); + // Lower bound only, for the same reason as the Retry-After checks. + Support.check(elapsedMs >= 1000, "the retry came after " + elapsedMs + " ms, sooner than the 1 s first backoff step"); + Support.note("the call took " + elapsedMs + " ms; only the 1 s lower bound is asserted, which is weak evidence"); + + Support.step("listing"); + Support.check(names.contains(name), "the listing did not contain " + name + ": " + names); + }); + } +} diff --git a/.sdkharness/tests/resilience/ApiRetryAfter429.java b/.sdkharness/tests/resilience/ApiRetryAfter429.java new file mode 100644 index 000000000..dc7a9a2c2 --- /dev/null +++ b/.sdkharness/tests/resilience/ApiRetryAfter429.java @@ -0,0 +1,45 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import java.util.ArrayList; +import java.util.List; + +/** + * api.retry_after_429. Run by ../run-resilience (see ../../README.md). + * + *

A 429 with Retry-After: 2 is not retried sooner than 2 seconds later (a lower bound on the whole + * call), and the listing then succeeds. + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class ApiRetryAfter429 { + public static void main(String[] args) { + Support.run("api.retry_after_429", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/api-retry_after_429.bin"; + Support.upload(name, Support.payload(1024, 429)); + + Support.step("arm the fault"); + Support.armFault("b2_list_file_names", 429, "too_many_requests", 2); + + Support.step("list"); + Support.markJournal(); + final List names = new ArrayList<>(); + final long elapsedMs = Support.timed(() -> names.addAll(Support.listNames())); + + Support.step("recovery path"); + final List window = Support.only(Support.sinceMark(), "b2_list_file_names"); + Support.check(window.size() >= 2 && window.get(0).status == 429 && window.get(0).fault.equals("injected"), + "the first b2_list_file_names did not carry the 429 fault: " + Support.show(window)); + Support.check(window.get(window.size() - 1).status == 200, "the listing was never retried to a 200: " + Support.show(window)); + + Support.step("retry-after"); + // Lower bound only: the journal has no timestamps, so the whole call is timed, and an upper bound would be flaky. + Support.check(elapsedMs >= 2000, "the retry came after " + elapsedMs + " ms, sooner than the 2 s Retry-After"); + Support.note("the call took " + elapsedMs + " ms; only the 2 s lower bound is asserted, which is weak evidence of honouring Retry-After"); + + Support.step("listing"); + Support.check(names.contains(name), "the listing did not contain " + name + ": " + names); + }); + } +} diff --git a/.sdkharness/tests/resilience/ApiRetryAfter503.java b/.sdkharness/tests/resilience/ApiRetryAfter503.java new file mode 100644 index 000000000..9003a4e80 --- /dev/null +++ b/.sdkharness/tests/resilience/ApiRetryAfter503.java @@ -0,0 +1,45 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import java.util.ArrayList; +import java.util.List; + +/** + * api.retry_after_503. Run by ../run-resilience (see ../../README.md). + * + *

A 503 with Retry-After: 2 is not retried sooner than 2 seconds later (a lower bound on the whole + * call), and the listing then succeeds. + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class ApiRetryAfter503 { + public static void main(String[] args) { + Support.run("api.retry_after_503", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/api-retry_after_503.bin"; + Support.upload(name, Support.payload(1024, 503)); + + Support.step("arm the fault"); + Support.armFault("b2_list_file_names", 503, "service_unavailable", 2); + + Support.step("list"); + Support.markJournal(); + final List names = new ArrayList<>(); + final long elapsedMs = Support.timed(() -> names.addAll(Support.listNames())); + + Support.step("recovery path"); + final List window = Support.only(Support.sinceMark(), "b2_list_file_names"); + Support.check(window.size() >= 2 && window.get(0).status == 503 && window.get(0).fault.equals("injected"), + "the first b2_list_file_names did not carry the 503 fault: " + Support.show(window)); + Support.check(window.get(window.size() - 1).status == 200, "the listing was never retried to a 200: " + Support.show(window)); + + Support.step("retry-after"); + // Lower bound only: the journal has no timestamps, so the whole call is timed, and an upper bound would be flaky. + Support.check(elapsedMs >= 2000, "the retry came after " + elapsedMs + " ms, sooner than the 2 s Retry-After"); + Support.note("the call took " + elapsedMs + " ms; only the 2 s lower bound is asserted, which is weak evidence of honouring Retry-After"); + + Support.step("listing"); + Support.check(names.contains(name), "the listing did not contain " + name + ": " + names); + }); + } +} diff --git a/.sdkharness/tests/resilience/AuthClockExpiry.java b/.sdkharness/tests/resilience/AuthClockExpiry.java new file mode 100644 index 000000000..725622c35 --- /dev/null +++ b/.sdkharness/tests/resilience/AuthClockExpiry.java @@ -0,0 +1,47 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import java.util.List; + +/** + * auth.clock_expiry. Run by ../run-resilience (see ../../README.md). + * + *

One long-lived client (B2StorageClient is one) keeps working past the 24 h life of its account token: + * the simulator's clock is moved a day and a millisecond forward, the real 401 comes back, and the + * client calls b2_authorize_account again and the listing succeeds. No fault is injected; the 401 is the + * simulator's own. + * https://www.backblaze.com/apidocs/b2-authorize-account + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class AuthClockExpiry { + public static void main(String[] args) { + Support.run("auth.clock_expiry", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/auth-clock-expiry.bin"; + Support.upload(name, Support.payload(1024, 12)); + // One call first, so the client is holding the token that is about to expire. + Support.check(Support.listNames().contains(name), "the listing before the clock moved did not contain " + name); + + Support.step("move the clock"); + Support.advanceClock(86400001L); + + Support.step("list"); + Support.markJournal(); + final List names = Support.listNames(); + + Support.step("recovery path"); + final List window = Support.sinceMark(); + final int hit = Support.indexOf(window, 0, "b2_list_file_names"); + Support.check(hit >= 0 && window.get(hit).status == 401 && window.get(hit).fault.isEmpty(), + "the first b2_list_file_names was not the simulator's own 401: " + Support.show(window)); + final int auth = Support.indexOf(window, hit + 1, "b2_authorize_account"); + Support.check(auth >= 0 && window.get(auth).status == 200, "no b2_authorize_account after the 401: " + Support.show(window)); + final int retry = Support.indexOf(window, auth + 1, "b2_list_file_names"); + Support.check(retry >= 0 && window.get(retry).status == 200, "no successful b2_list_file_names after reauthorizing: " + Support.show(window)); + + Support.step("listing"); + Support.check(names.contains(name), "the listing did not contain " + name + ": " + names); + }); + } +} diff --git a/.sdkharness/tests/resilience/AuthExpired401.java b/.sdkharness/tests/resilience/AuthExpired401.java new file mode 100644 index 000000000..34178b841 --- /dev/null +++ b/.sdkharness/tests/resilience/AuthExpired401.java @@ -0,0 +1,43 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import java.util.List; + +/** + * auth.expired_401. Run by ../run-resilience (see ../../README.md). + * + *

A 401 expired_auth_token on an API call is answered by b2_authorize_account again, and the listing then + * succeeds. The journal shows the faulted b2_list_file_names, an authorize, and a successful listing. + * https://www.backblaze.com/apidocs/b2-list-file-names + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class AuthExpired401 { + public static void main(String[] args) { + Support.run("auth.expired_401", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/auth-expired-401.bin"; + Support.upload(name, Support.payload(1024, 11)); + + Support.step("arm the fault"); + Support.armFault("b2_list_file_names", 401, "expired_auth_token", null); + + Support.step("list"); + Support.markJournal(); + final List names = Support.listNames(); + + Support.step("recovery path"); + final List window = Support.sinceMark(); + final int hit = Support.indexOf(window, 0, "b2_list_file_names"); + Support.check(hit >= 0 && window.get(hit).status == 401 && window.get(hit).fault.equals("injected"), + "the first b2_list_file_names did not carry the 401 fault: " + Support.show(window)); + final int auth = Support.indexOf(window, hit + 1, "b2_authorize_account"); + Support.check(auth >= 0 && window.get(auth).status == 200, "no b2_authorize_account after the 401: " + Support.show(window)); + final int retry = Support.indexOf(window, auth + 1, "b2_list_file_names"); + Support.check(retry >= 0 && window.get(retry).status == 200, "no successful b2_list_file_names after reauthorizing: " + Support.show(window)); + + Support.step("listing"); + Support.check(names.contains(name), "the listing did not contain " + name + ": " + names); + }); + } +} diff --git a/.sdkharness/tests/resilience/DownloadRetry503.java b/.sdkharness/tests/resilience/DownloadRetry503.java new file mode 100644 index 000000000..e9459f715 --- /dev/null +++ b/.sdkharness/tests/resilience/DownloadRetry503.java @@ -0,0 +1,39 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import java.util.Arrays; +import java.util.List; + +/** + * download.retry_503. Run by ../run-resilience (see ../../README.md). + * + *

A 503 on b2_download_file_by_id is retried, and the bytes delivered are intact. + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class DownloadRetry503 { + public static void main(String[] args) { + Support.run("download.retry_503", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/download-retry-503.bin"; + final byte[] data = Support.payload(4096, 14); + final String fileId = Support.upload(name, data).getFileId(); + + Support.step("arm the fault"); + Support.armFault("b2_download_file_by_id", 503, "service_unavailable", null); + + Support.step("download"); + Support.markJournal(); + final byte[] got = Support.downloadById(fileId); + + Support.step("recovery path"); + final List window = Support.only(Support.sinceMark(), "b2_download_file_by_id"); + Support.check(window.size() >= 2 && window.get(0).status == 503 && window.get(0).fault.equals("injected"), + "the first b2_download_file_by_id did not carry the 503 fault: " + Support.show(window)); + Support.check(window.get(window.size() - 1).status == 200, "the download was never retried to a 200: " + Support.show(window)); + + Support.step("bytes"); + Support.check(Arrays.equals(data, got), "the downloaded " + got.length + " bytes differ from the " + data.length + " uploaded"); + }); + } +} diff --git a/.sdkharness/tests/resilience/PartRetry503.java b/.sdkharness/tests/resilience/PartRetry503.java new file mode 100644 index 000000000..a3efcf1c0 --- /dev/null +++ b/.sdkharness/tests/resilience/PartRetry503.java @@ -0,0 +1,54 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import java.util.List; + +/** + * part.retry_503. Run by ../run-resilience (see ../../README.md). + * + *

A 503 on b2_upload_part is retried: the part goes to a different upload part URL, and the large file + * finishes with its bytes intact. The journal shows the faulted part, a successful b2_upload_part on a + * different URL id, and b2_finish_large_file. + * https://www.backblaze.com/apidocs/b2-upload-part + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class PartRetry503 { + public static void main(String[] args) { + Support.run("part.retry_503", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/part-retry-503.bin"; + // Several parts at the simulator's 5000-byte part size, so a part can be retried while others proceed. + final byte[] data = Support.payload(16000, 10); + + Support.step("arm the fault"); + Support.armFault("b2_upload_part", 503, "service_unavailable", null); + + Support.step("upload"); + Support.markJournal(); + Support.uploadLarge(name, data); + + Support.step("recovery path"); + final List window = Support.sinceMark(); + final List parts = Support.only(window, "b2_upload_part"); + Support.Entry faulted = null; + for (Support.Entry e : parts) { + if (e.fault.equals("injected") && e.status == 503) { + faulted = e; + } + } + Support.check(faulted != null, "no b2_upload_part carried the 503 fault: " + Support.show(window)); + int later = 0; + for (Support.Entry e : parts.subList(parts.indexOf(faulted) + 1, parts.size())) { + if (e.status == 200 && !e.uploadUrlId.isEmpty() && !e.uploadUrlId.equals(faulted.uploadUrlId)) { + later++; + } + } + Support.check(later > 0, "no b2_upload_part succeeded on a part URL other than the faulted one (" + faulted.uploadUrlId + "): " + Support.show(window)); + Support.check(Support.indexOf(window, 0, "b2_finish_large_file") >= 0, "the large file was never finished: " + Support.show(window)); + + Support.step("read back"); + Support.requireIntact(name, data); + }); + } +} diff --git a/.sdkharness/tests/resilience/Support.java b/.sdkharness/tests/resilience/Support.java new file mode 100644 index 000000000..f57d03654 --- /dev/null +++ b/.sdkharness/tests/resilience/Support.java @@ -0,0 +1,437 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.B2ClientConfig; +import com.backblaze.b2.client.B2StorageClient; +import com.backblaze.b2.client.contentHandlers.B2ContentMemoryWriter; +import com.backblaze.b2.client.contentSources.B2ByteArrayContentSource; +import com.backblaze.b2.client.exceptions.B2Exception; +import com.backblaze.b2.client.structures.B2Bucket; +import com.backblaze.b2.client.structures.B2FileVersion; +import com.backblaze.b2.client.structures.B2ListFileNamesRequest; +import com.backblaze.b2.client.structures.B2ListFileVersionsRequest; +import com.backblaze.b2.client.structures.B2UploadFileRequest; +import com.backblaze.b2.client.webApiHttpClient.B2StorageHttpClientBuilder; +import com.backblaze.b2.client.webApiHttpClient.HttpClientFactoryImpl; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Shared plumbing for the sdkharness resilience checks. Compiled together with one check class by + * ../run-resilience (a JDK 11 single-file launch cannot import a second source file). + * + *

Each check is one {@code main} that calls {@link #run}. A check arms one fault through the + * simulator's control listener, makes one SDK call, and asserts both the end state (the bytes read + * back, the listing, the error) and the recovery path, which it reads from the simulator's request + * journal ({@code GET /journal}: sequence, endpoint, status, fault, an opaque upload URL id; never a + * header, token or body). {@link #run} prints exactly one result line, + * {@code RESILIENCE b2-sdk-java @simulator: PASS | FAIL ( -- )}, after it + * has cleared every fault and deleted every bucket and file the check created. Anything else it + * prints is a {@code NOTE} line. It never prints a response body or a credential. + */ +public final class Support { + static final String SLUG = "b2-sdk-java"; + // The simulator's fixed test credential. No other credential is ever used. + private static final String KEY_ID = "test-key-id"; + private static final String KEY = "test-key"; + private static final String LOOPBACK = "http://127\\.0\\.0\\.1:[0-9]{1,5}/?"; + + /** The body of one check. */ + interface Body { + void run() throws Exception; + } + + /** One SDK call whose duration the check wants to observe. */ + interface Call { + void run() throws Exception; + } + + /** A failed assertion; the step is whatever {@link #step} last named. */ + static final class Failure extends RuntimeException { + Failure(String detail) { + super(detail); + } + } + + /** One line of the simulator's request journal. */ + static final class Entry { + final String endpoint; + /** The HTTP status, or -1 when the simulator closed the connection before answering. */ + final int status; + /** The fault that fired on this request ("injected", "stall", "reset-mid-request", ...), or "". */ + final String fault; + /** The opaque upload (or part) URL id the request used, or "". */ + final String uploadUrlId; + + Entry(String endpoint, int status, String fault, String uploadUrlId) { + this.endpoint = endpoint; + this.status = status; + this.fault = fault; + this.uploadUrlId = uploadUrlId; + } + + @Override + public String toString() { + return endpoint + ":" + (status < 0 ? "none" : String.valueOf(status)) + + (fault.isEmpty() ? "" : "(" + fault + ")") + (uploadUrlId.isEmpty() ? "" : "@" + uploadUrlId); + } + } + + static B2StorageClient client; + static String bucketId; + static String bucketName; + + private static String control; + private static final HttpClient http = HttpClient.newHttpClient(); + private static String scenario = "unknown"; + private static String step = "configuration"; + private static final List created = new ArrayList<>(); + private static int journalMark; + + private Support() { + } + + // ---- the harness around one check ---------------------------------------------------------- + + /** Runs a check with the SDK's default HTTP settings. */ + static void run(String id, Body body) { + run(id, 0, body); + } + + /** + * Runs a check. {@code socketTimeoutSeconds} above zero sets the SDK's documented per-request + * socket timeout (HttpClientFactoryImpl.Builder.setSocketTimeoutSeconds, whole seconds); zero + * leaves the default. + */ + static void run(String id, int socketTimeoutSeconds, Body body) { + scenario = id; + String failure = null; + try { + final String url = System.getenv("SDKHARNESS_SIMULATOR_URL"); + control = System.getenv("SDKHARNESS_SIMULATOR_CONTROL_URL"); + // run-resilience checked both already; the SDK's http switch must never reach anything else. + if (url == null || !url.matches(LOOPBACK) || control == null || !control.matches(LOOPBACK)) { + throw new Failure("the simulator and control URLs must be IPv4 loopback HTTP origins"); + } + control = control.replaceAll("/$", ""); + client = newClient(url, socketTimeoutSeconds); + body.run(); + } catch (Failure f) { + failure = step + " -- " + tidy(f.getMessage()); + } catch (Throwable t) { + failure = step + " -- " + describe(t); + } finally { + cleanup(); + } + if (failure == null) { + System.out.println("RESILIENCE " + SLUG + " " + scenario + " @simulator: PASS"); + } else { + System.out.println("RESILIENCE " + SLUG + " " + scenario + " @simulator: FAIL (" + failure + ")"); + System.exit(1); + } + } + + private static B2StorageClient newClient(String masterUrl, int socketTimeoutSeconds) { + final B2ClientConfig config = B2ClientConfig.builder(KEY_ID, KEY, "b2-sdk-java-sdkharness-resilience") + .setMasterUrl(masterUrl) + .build(); + // The simulator speaks plain HTTP on loopback, so this enables the SDK's documented + // test-environment switch; the SDK's own retry policy and every other default stay as shipped. + final HttpClientFactoryImpl.Builder httpBuilder = HttpClientFactoryImpl.builder().setSupportInsecureHttp(true); + if (socketTimeoutSeconds > 0) { + httpBuilder.setSocketTimeoutSeconds(socketTimeoutSeconds); + } + return B2StorageHttpClientBuilder.builder(config).setHttpClientFactory(httpBuilder.build()).build(); + } + + /** + * Clears every fault still armed (so the cleanup calls below cannot trip one), then deletes every + * bucket and file this check created, on the error path too. + */ + private static void cleanup() { + try { + if (control != null) { + send("DELETE", "/faults", null); + } + } catch (Exception e) { + note("clearing the faults failed: " + describe(e)); + } + if (client == null) { + return; + } + for (B2Bucket bucket : created) { + try { + client.deleteAllFilesInBucket(bucket.getBucketId()); + client.deleteBucket(bucket.getBucketId()); + } catch (B2Exception | RuntimeException e) { + note("cleanup of " + bucket.getBucketName() + " failed: " + describe(e)); + } + } + try { + client.close(); + } catch (RuntimeException e) { + // nothing left to release + } + } + + // ---- reporting ------------------------------------------------------------------------------ + + static void step(String name) { + step = name; + } + + static void note(String text) { + System.out.println("NOTE " + SLUG + " " + scenario + ": " + tidy(text)); + } + + static void fail(String detail) { + throw new Failure(detail); + } + + static void check(boolean ok, String detail) { + if (!ok) { + fail(detail); + } + } + + /** One line, printable, capped: a detail never carries a response body or a multi-line diagnostic. */ + static String tidy(String s) { + final String line = String.valueOf(s).replaceAll("\\s+", " ").replaceAll("[^\\x20-\\x7E]", "?"); + return line.length() > 200 ? line.substring(0, 200) : line; + } + + static String describe(Throwable t) { + final StringBuilder sb = new StringBuilder(t.getClass().getSimpleName()); + if (t instanceof B2Exception) { + sb.append(" status=").append(((B2Exception) t).getStatus()).append(" code=").append(((B2Exception) t).getCode()); + } else if (t.getMessage() != null) { + sb.append(' ').append(t.getMessage()); + } + return tidy(sb.toString()); + } + + // ---- the simulator's control listener ------------------------------------------------------ + + private static String send(String method, String path, String json) throws Exception { + final HttpRequest.Builder request = HttpRequest.newBuilder(URI.create(control + path)); + if (json == null) { + request.method(method, HttpRequest.BodyPublishers.noBody()); + } else { + request.header("Content-Type", "application/json").method(method, HttpRequest.BodyPublishers.ofString(json)); + } + final HttpResponse response = http.send(request.build(), HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() / 100 != 2) { + fail("the control listener answered " + response.statusCode() + " to " + method + " " + path); + } + return response.body(); + } + + /** Arms one status fault on {@code endpoint}: {@code POST /faults {on, status, code, retryAfter?, count: 1}}. */ + static void armFault(String endpoint, int status, String code, Integer retryAfterSeconds) throws Exception { + send("POST", "/faults", "{\"on\":\"" + endpoint + "\",\"status\":" + status + ",\"code\":\"" + code + "\"" + + (retryAfterSeconds == null ? "" : ",\"retryAfter\":" + retryAfterSeconds) + ",\"count\":1}"); + } + + /** Arms one connection fault on {@code endpoint}: {@code POST /wire-faults {on, kind, ms?, count: 1}}. */ + static void armWireFault(String endpoint, String kind, Integer ms) throws Exception { + send("POST", "/wire-faults", "{\"on\":\"" + endpoint + "\",\"kind\":\"" + kind + "\"" + + (ms == null ? "" : ",\"ms\":" + ms) + ",\"count\":1}"); + } + + /** Moves the simulator's clock forward: {@code POST /clock {advanceMs}}. */ + static void advanceClock(long ms) throws Exception { + send("POST", "/clock", "{\"advanceMs\":" + ms + "}"); + } + + private static final Pattern ENTRY = Pattern.compile("\\{[^{}]*\"endpoint\"[^{}]*\\}"); + + private static List journal() throws Exception { + final List entries = new ArrayList<>(); + final Matcher m = ENTRY.matcher(send("GET", "/journal", null)); + while (m.find()) { + final String o = m.group(); + entries.add(new Entry(text(o, "endpoint"), number(o, "status"), text(o, "fault"), text(o, "uploadUrlId"))); + } + return entries; + } + + private static String text(String object, String key) { + final Matcher m = Pattern.compile("\"" + key + "\"\\s*:\\s*(null|\"([^\"]*)\")").matcher(object); + return m.find() && m.group(2) != null ? m.group(2) : ""; + } + + private static int number(String object, String key) { + final Matcher m = Pattern.compile("\"" + key + "\"\\s*:\\s*(-?[0-9]+)").matcher(object); + return m.find() ? Integer.parseInt(m.group(1)) : -1; + } + + /** Remembers where the journal stands, so {@link #sinceMark} returns only what the call under test caused. */ + static void markJournal() throws Exception { + journalMark = journal().size(); + } + + /** The requests the simulator saw since {@link #markJournal}, in order. */ + static List sinceMark() throws Exception { + final List all = journal(); + check(all.size() >= journalMark, "the journal shrank (" + all.size() + " entries after " + journalMark + ")"); + return new ArrayList<>(all.subList(journalMark, all.size())); + } + + static List only(List entries, String endpoint) { + final List out = new ArrayList<>(); + for (Entry e : entries) { + if (e.endpoint.equals(endpoint)) { + out.add(e); + } + } + return out; + } + + static String show(List entries) { + final StringBuilder sb = new StringBuilder(); + for (Entry e : entries) { + sb.append(sb.length() == 0 ? "" : " ").append(e); + } + return tidy(sb.toString()); + } + + /** The index of the first entry at or after {@code from} on {@code endpoint}, or -1. */ + static int indexOf(List entries, int from, String endpoint) { + for (int i = from; i < entries.size(); i++) { + if (entries.get(i).endpoint.equals(endpoint)) { + return i; + } + } + return -1; + } + + // ---- fixtures ------------------------------------------------------------------------------- + + /** A fresh private bucket named sdkharness-res-, deleted (with its files) on exit. */ + static B2Bucket bucket() throws B2Exception { + final B2Bucket bucket = client.createBucket( + "sdkharness-res-" + UUID.randomUUID().toString().replace("-", "").substring(0, 12), "allPrivate"); + created.add(bucket); + bucketId = bucket.getBucketId(); + bucketName = bucket.getBucketName(); + return bucket; + } + + static B2FileVersion upload(String name, byte[] data) throws B2Exception { + return client.uploadSmallFile(B2UploadFileRequest + .builder(bucketId, name, "application/octet-stream", B2ByteArrayContentSource.build(data)).build()); + } + + /** Uploads {@code data} as a large file in several parts, two at a time. */ + static B2FileVersion uploadLarge(String name, byte[] data) throws B2Exception { + final ExecutorService executor = Executors.newFixedThreadPool(2); + try { + return client.uploadLargeFile(B2UploadFileRequest + .builder(bucketId, name, "application/octet-stream", B2ByteArrayContentSource.build(data)).build(), executor); + } finally { + executor.shutdown(); + } + } + + /** Deterministic bytes that cover all 256 values, so a text-only path cannot pass. */ + static byte[] payload(int length, long seed) { + final byte[] data = new byte[length]; + new Random(seed).nextBytes(data); + for (int i = 0; i < Math.min(256, length); i++) { + data[i] = (byte) i; + } + return data; + } + + static byte[] downloadById(String fileId) throws B2Exception { + final B2ContentMemoryWriter writer = B2ContentMemoryWriter.build(); + client.downloadById(fileId, writer); + return writer.getBytes(); + } + + static byte[] downloadByName(String name) throws B2Exception { + final B2ContentMemoryWriter writer = B2ContentMemoryWriter.build(); + client.downloadByName(bucketName, name, writer); + return writer.getBytes(); + } + + /** Every version of {@code name} in the check's bucket, as the listing returns them. */ + static List versionsOf(String name) throws B2Exception { + final List all = new ArrayList<>(); + for (B2FileVersion v : client.fileVersions(B2ListFileVersionsRequest.builder(bucketId).setPrefix(name).build())) { + if (v.getFileName().equals(name)) { + all.add(v); + } + } + return all; + } + + static List listNames() throws B2Exception { + final List names = new ArrayList<>(); + for (B2FileVersion v : client.fileNames(B2ListFileNamesRequest.builder(bucketId).build())) { + names.add(v.getFileName()); + } + return names; + } + + /** Reads {@code name} back by name and requires it to be byte-for-byte {@code expected}. */ + static void requireIntact(String name, byte[] expected) throws B2Exception { + final byte[] got = downloadByName(name); + check(Arrays.equals(expected, got), + name + " read back as " + got.length + " bytes that differ from the " + expected.length + " uploaded"); + } + + static String sha1(byte[] data) { + try { + final StringBuilder hex = new StringBuilder(); + for (byte b : MessageDigest.getInstance("SHA-1").digest(data)) { + hex.append(String.format("%02x", b)); + } + return hex.toString(); + } catch (java.security.NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + /** Runs {@code call} and returns how long it took, in milliseconds (a lower bound is all a check may assert). */ + static long timed(Call call) throws Exception { + final long start = System.nanoTime(); + call.run(); + return (System.nanoTime() - start) / 1_000_000; + } + + // ---- shared assertions --------------------------------------------------------------------- + + /** + * The upload recovery path every {@code upload.*} status and wire fault shares: after the faulted + * {@code b2_upload_file} the SDK asks for a fresh upload URL ({@code b2_get_upload_url}), and the + * retried {@code b2_upload_file} succeeds on a different URL id. + */ + static void requireFreshUrlRetry(String fault, List window) { + final int hit = indexOf(window, 0, "b2_upload_file"); + check(hit >= 0 && window.get(hit).fault.equals(fault), + "the first b2_upload_file did not carry the " + fault + " fault: " + show(window)); + final int again = indexOf(window, hit + 1, "b2_get_upload_url"); + check(again >= 0, "no fresh b2_get_upload_url after the faulted upload: " + show(window)); + final int retry = indexOf(window, again + 1, "b2_upload_file"); + check(retry >= 0, "no second b2_upload_file after the fresh upload URL: " + show(window)); + final Entry first = window.get(hit); + final Entry second = window.get(retry); + check(second.status == 200 && second.fault.isEmpty(), "the retried upload did not succeed: " + show(window)); + check(!second.uploadUrlId.isEmpty() && !second.uploadUrlId.equals(first.uploadUrlId), + "the retry reused the faulted upload URL (" + first.uploadUrlId + "): " + show(window)); + } +} diff --git a/.sdkharness/tests/resilience/UploadCapExceeded403.java b/.sdkharness/tests/resilience/UploadCapExceeded403.java new file mode 100644 index 000000000..2f400f94d --- /dev/null +++ b/.sdkharness/tests/resilience/UploadCapExceeded403.java @@ -0,0 +1,47 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.exceptions.B2Exception; +import java.util.List; + +/** + * upload.cap_exceeded_403. Run by ../run-resilience (see ../../README.md). + * + *

A 403 cap_exceeded on b2_upload_file is a hard stop, not a transient fault: the error reaches the + * caller and the upload is not retried. The journal shows exactly one b2_upload_file, and nothing + * was stored. + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class UploadCapExceeded403 { + public static void main(String[] args) { + Support.run("upload.cap_exceeded_403", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/upload-cap-exceeded-403.bin"; + + Support.step("arm the fault"); + Support.armFault("b2_upload_file", 403, "cap_exceeded", null); + + Support.step("upload"); + Support.markJournal(); + B2Exception error = null; + try { + Support.upload(name, Support.payload(2048, 403)); + } catch (B2Exception e) { + error = e; + } + Support.check(error != null, "the upload succeeded although the simulator answered 403 cap_exceeded"); + Support.check(error.getStatus() == 403, "the error that reached the caller was not a 403: " + Support.describe(error)); + Support.check("cap_exceeded".equals(error.getCode()), "the error that reached the caller lost its code: " + Support.describe(error)); + + Support.step("recovery path"); + final List window = Support.sinceMark(); + final List uploads = Support.only(window, "b2_upload_file"); + Support.check(uploads.size() == 1, "expected exactly one b2_upload_file (no retry), got " + uploads.size() + ": " + Support.show(window)); + Support.check(uploads.get(0).status == 403 && uploads.get(0).fault.equals("injected"), "the single upload was not the faulted one: " + Support.show(window)); + + Support.step("nothing stored"); + Support.check(Support.versionsOf(name).isEmpty(), "a version of " + name + " exists although the upload was refused"); + }); + } +} diff --git a/.sdkharness/tests/resilience/UploadExpiredToken401.java b/.sdkharness/tests/resilience/UploadExpiredToken401.java new file mode 100644 index 000000000..be1ef564b --- /dev/null +++ b/.sdkharness/tests/resilience/UploadExpiredToken401.java @@ -0,0 +1,36 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +/** + * upload.expired_token_401. Run by ../run-resilience (see ../../README.md). + * + *

A 401 on b2_upload_file (the upload authorization token expired) is answered by b2_get_upload_url + * again, then the file lands intact. This SDK takes a new upload URL for it and does not + * reauthorize the account, which is the documented path for an upload URL token. + * https://www.backblaze.com/apidocs/b2-upload-file + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class UploadExpiredToken401 { + public static void main(String[] args) { + Support.run("upload.expired_token_401", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/upload-expired_token_401.bin"; + final byte[] data = Support.payload(2048, 401); + + Support.step("arm the fault"); + Support.armFault("b2_upload_file", 401, "expired_auth_token", null); + + Support.step("upload"); + Support.markJournal(); + Support.upload(name, data); + + Support.step("recovery path"); + Support.requireFreshUrlRetry("injected", Support.sinceMark()); + + Support.step("read back"); + Support.requireIntact(name, data); + Support.check(Support.versionsOf(name).size() == 1, "expected exactly one stored version of " + name); + }); + } +} diff --git a/.sdkharness/tests/resilience/UploadGetUrl503.java b/.sdkharness/tests/resilience/UploadGetUrl503.java new file mode 100644 index 000000000..a285cb2ec --- /dev/null +++ b/.sdkharness/tests/resilience/UploadGetUrl503.java @@ -0,0 +1,45 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import java.util.List; + +/** + * upload.get_url_503. Run by ../run-resilience (see ../../README.md). + * + *

A 503 on b2_get_upload_url is retried, then the file lands intact. The fault is armed before the + * first upload in the bucket: the SDK pools upload URLs, so after any earlier upload it would reuse one + * and never call b2_get_upload_url at all (the harness then rejects a PASS its journal does not back). + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class UploadGetUrl503 { + public static void main(String[] args) { + Support.run("upload.get_url_503", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/upload-get-url-503.bin"; + final byte[] data = Support.payload(2048, 5); + + Support.step("arm the fault"); + // Before the first upload in this bucket: nothing is pooled yet, so the first upload must ask for a URL. + Support.armFault("b2_get_upload_url", 503, "service_unavailable", null); + + Support.step("upload"); + Support.markJournal(); + Support.upload(name, data); + + Support.step("recovery path"); + final List window = Support.sinceMark(); + final int hit = Support.indexOf(window, 0, "b2_get_upload_url"); + Support.check(hit >= 0 && window.get(hit).status == 503 && window.get(hit).fault.equals("injected"), + "the first b2_get_upload_url did not carry the 503 fault: " + Support.show(window)); + final int again = Support.indexOf(window, hit + 1, "b2_get_upload_url"); + Support.check(again >= 0 && window.get(again).status == 200, + "b2_get_upload_url was not retried to a 200: " + Support.show(window)); + final int sent = Support.indexOf(window, again + 1, "b2_upload_file"); + Support.check(sent >= 0 && window.get(sent).status == 200, "no successful b2_upload_file after the retry: " + Support.show(window)); + + Support.step("read back"); + Support.requireIntact(name, data); + }); + } +} diff --git a/.sdkharness/tests/resilience/UploadResetBeforeResponse.java b/.sdkharness/tests/resilience/UploadResetBeforeResponse.java new file mode 100644 index 000000000..1ac8ff99d --- /dev/null +++ b/.sdkharness/tests/resilience/UploadResetBeforeResponse.java @@ -0,0 +1,39 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +/** + * upload.reset_before_response. Run by ../run-resilience (see ../../README.md). + * + *

The connection is reset after the server has stored the file but before it answers, so the client + * cannot know. It must retry on a fresh upload URL, and the file reads back intact; a second version of + * the same name is acceptable (the first upload did land). + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class UploadResetBeforeResponse { + public static void main(String[] args) { + Support.run("upload.reset_before_response", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/upload-reset-before-response.bin"; + final byte[] data = Support.payload(4096, 8); + + Support.step("arm the fault"); + Support.armWireFault("b2_upload_file", "reset-before-response", null); + + Support.step("upload"); + Support.markJournal(); + Support.upload(name, data); + + Support.step("recovery path"); + Support.requireFreshUrlRetry("reset-before-response", Support.sinceMark()); + + Support.step("read back"); + Support.requireIntact(name, data); + final int versions = Support.versionsOf(name).size(); + Support.check(versions == 1 || versions == 2, "expected one or two stored versions of " + name + ", got " + versions); + if (versions == 2) { + Support.note("two versions of the same name: the reset came after the first upload was stored, which the scenario accepts"); + } + }); + } +} diff --git a/.sdkharness/tests/resilience/UploadResetMidRequest.java b/.sdkharness/tests/resilience/UploadResetMidRequest.java new file mode 100644 index 000000000..ceb60dfa8 --- /dev/null +++ b/.sdkharness/tests/resilience/UploadResetMidRequest.java @@ -0,0 +1,34 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +/** + * upload.reset_mid_request. Run by ../run-resilience (see ../../README.md). + * + *

The connection is reset while the body is being sent. The client treats it as a failed upload, asks for + * a fresh upload URL and retries; the file lands intact. + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class UploadResetMidRequest { + public static void main(String[] args) { + Support.run("upload.reset_mid_request", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/upload-reset-mid-request.bin"; + final byte[] data = Support.payload(4096, 7); + + Support.step("arm the fault"); + Support.armWireFault("b2_upload_file", "reset-mid-request", null); + + Support.step("upload"); + Support.markJournal(); + Support.upload(name, data); + + Support.step("recovery path"); + Support.requireFreshUrlRetry("reset-mid-request", Support.sinceMark()); + + Support.step("read back"); + Support.requireIntact(name, data); + Support.check(Support.versionsOf(name).size() == 1, "expected exactly one stored version of " + name); + }); + } +} diff --git a/.sdkharness/tests/resilience/UploadRetry408.java b/.sdkharness/tests/resilience/UploadRetry408.java new file mode 100644 index 000000000..8bae74a05 --- /dev/null +++ b/.sdkharness/tests/resilience/UploadRetry408.java @@ -0,0 +1,34 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +/** + * upload.retry_408. Run by ../run-resilience (see ../../README.md). + * + *

A 408 on b2_upload_file is retried the same way: a fresh upload URL, then the file lands intact. + * https://www.backblaze.com/apidocs/b2-upload-file + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class UploadRetry408 { + public static void main(String[] args) { + Support.run("upload.retry_408", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/upload-retry_408.bin"; + final byte[] data = Support.payload(2048, 408); + + Support.step("arm the fault"); + Support.armFault("b2_upload_file", 408, "request_timeout", null); + + Support.step("upload"); + Support.markJournal(); + Support.upload(name, data); + + Support.step("recovery path"); + Support.requireFreshUrlRetry("injected", Support.sinceMark()); + + Support.step("read back"); + Support.requireIntact(name, data); + Support.check(Support.versionsOf(name).size() == 1, "expected exactly one stored version of " + name); + }); + } +} diff --git a/.sdkharness/tests/resilience/UploadRetry500.java b/.sdkharness/tests/resilience/UploadRetry500.java new file mode 100644 index 000000000..4fabfffa6 --- /dev/null +++ b/.sdkharness/tests/resilience/UploadRetry500.java @@ -0,0 +1,34 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +/** + * upload.retry_500. Run by ../run-resilience (see ../../README.md). + * + *

A 500 on b2_upload_file is retried the same way as a 503: a fresh upload URL, then the file lands + * intact. Asserted on the 500 status only, never its code (no page sources one). + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class UploadRetry500 { + public static void main(String[] args) { + Support.run("upload.retry_500", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/upload-retry_500.bin"; + final byte[] data = Support.payload(2048, 500); + + Support.step("arm the fault"); + Support.armFault("b2_upload_file", 500, "internal_error", null); + + Support.step("upload"); + Support.markJournal(); + Support.upload(name, data); + + Support.step("recovery path"); + Support.requireFreshUrlRetry("injected", Support.sinceMark()); + + Support.step("read back"); + Support.requireIntact(name, data); + Support.check(Support.versionsOf(name).size() == 1, "expected exactly one stored version of " + name); + }); + } +} diff --git a/.sdkharness/tests/resilience/UploadRetry503.java b/.sdkharness/tests/resilience/UploadRetry503.java new file mode 100644 index 000000000..50c40b96b --- /dev/null +++ b/.sdkharness/tests/resilience/UploadRetry503.java @@ -0,0 +1,36 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +/** + * upload.retry_503. Run by ../run-resilience (see ../../README.md). + * + *

A 503 on b2_upload_file means the upload URL is not usable right now: the client must ask + * b2_get_upload_url for a fresh one and retry, and the file must land intact. The journal shows the + * retried b2_upload_file on a new upload URL id. + * https://www.backblaze.com/apidocs/b2-upload-file + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class UploadRetry503 { + public static void main(String[] args) { + Support.run("upload.retry_503", () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/upload-retry_503.bin"; + final byte[] data = Support.payload(2048, 503); + + Support.step("arm the fault"); + Support.armFault("b2_upload_file", 503, "service_unavailable", null); + + Support.step("upload"); + Support.markJournal(); + Support.upload(name, data); + + Support.step("recovery path"); + Support.requireFreshUrlRetry("injected", Support.sinceMark()); + + Support.step("read back"); + Support.requireIntact(name, data); + Support.check(Support.versionsOf(name).size() == 1, "expected exactly one stored version of " + name); + }); + } +} diff --git a/.sdkharness/tests/resilience/UploadStall.java b/.sdkharness/tests/resilience/UploadStall.java new file mode 100644 index 000000000..975970a0c --- /dev/null +++ b/.sdkharness/tests/resilience/UploadStall.java @@ -0,0 +1,36 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +/** + * upload.stall. Run by ../run-resilience (see ../../README.md). + * + *

The server holds a connection open and sends nothing (60 s). A client with a request timeout must + * abandon the stalled request and retry on a fresh upload URL, and the file lands intact. This scenario + * needs a request timeout shorter than the stall: the SDK offers one, a documented builder option + * (HttpClientFactoryImpl.Builder.setSocketTimeoutSeconds), so the check sets 2 s and must PASS or + * FAIL, never no-client-option. The default is 20 s. + * https://www.backblaze.com/docs/cloud-storage-integration-checklist + */ +public class UploadStall { + public static void main(String[] args) { + Support.run("upload.stall", 2, () -> { + Support.step("setup"); + Support.bucket(); + final String name = "res/upload-stall.bin"; + final byte[] data = Support.payload(4096, 9); + + Support.step("arm the fault"); + Support.armWireFault("b2_upload_file", "stall", 60000); + + Support.step("upload"); + Support.markJournal(); + Support.upload(name, data); + + Support.step("recovery path"); + Support.requireFreshUrlRetry("stall", Support.sinceMark()); + + Support.step("read back"); + Support.requireIntact(name, data); + }); + } +} diff --git a/.sdkharness/tests/run-resilience b/.sdkharness/tests/run-resilience new file mode 100755 index 000000000..95b551f23 --- /dev/null +++ b/.sdkharness/tests/run-resilience @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Copyright 2026, Backblaze Inc. All Rights Reserved. +# License https://www.backblaze.com/using_b2_code.html +# +# sdkharness resilience (failure recovery) checks for this checkout: one scenario per run, named by +# SDKHARNESS_SCENARIO. See ../README.md. +# +# Each check is a Java program in resilience/ that shares resilience/Support.java. A JDK 11 +# single-file launch cannot import a second source file, so both are compiled here with javac +# against the SDK jars the harness built, then the check class is run. The check arms exactly one +# fault through the simulator's control listener (SDKHARNESS_SIMULATOR_CONTROL_URL) and asserts the +# recovery from the simulator's request journal. +# +# A check that outlives WATCHDOG_SECONDS is killed and reported as a FAIL: the slowest scenario +# (upload.stall, with its 2 s socket timeout) takes about 3 s, so a minute means the SDK never +# gave up on something. +set -u + +LEVEL=resilience +SLUG=b2-sdk-java +WATCHDOG_SECONDS=60 +SCENARIO="${SDKHARNESS_SCENARIO:-}" +HERE="$(cd "$(dirname "$0")" && pwd -P)" +result() { printf 'SDKHARNESS_RESULT\t%s\t%s\t%s\t%s\n' "$LEVEL" "$SCENARIO" "$1" "$2"; } +fail() { result FAIL "$1"; exit 1; } +# A detail is one printable line, capped: a compiler diagnostic never goes into a result as a block. +tidy() { printf '%s' "$1" | tr '\n\r\t' ' ' | tr -cd '[:print:]' | cut -c1-200; } + +[ "${SDKHARNESS_TEST_LEVEL:-}" = "$LEVEL" ] || fail 'configuration: unexpected test level' +case "$SCENARIO" in + upload.retry_503) class=UploadRetry503 ;; + upload.retry_500) class=UploadRetry500 ;; + upload.retry_408) class=UploadRetry408 ;; + upload.expired_token_401) class=UploadExpiredToken401 ;; + upload.get_url_503) class=UploadGetUrl503 ;; + upload.cap_exceeded_403) class=UploadCapExceeded403 ;; + upload.reset_mid_request) class=UploadResetMidRequest ;; + upload.reset_before_response) class=UploadResetBeforeResponse ;; + upload.stall) class=UploadStall ;; + part.retry_503) class=PartRetry503 ;; + auth.expired_401) class=AuthExpired401 ;; + auth.clock_expiry) class=AuthClockExpiry ;; + api.retry_after_429) class=ApiRetryAfter429 ;; + api.retry_after_503) class=ApiRetryAfter503 ;; + api.backoff_503) class=ApiBackoff503 ;; + download.retry_503) class=DownloadRetry503 ;; + *) fail 'configuration: no check for this scenario' ;; +esac +[[ "${SDKHARNESS_SIMULATOR_URL:-}" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/?$ ]] \ + || fail 'configuration: simulator URL must be an IPv4 loopback HTTP origin' +[[ "${SDKHARNESS_SIMULATOR_CONTROL_URL:-}" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/?$ ]] \ + || fail 'configuration: simulator control URL must be an IPv4 loopback HTTP origin' +[ -n "${SDKHARNESS_JAVA_CLASSPATH:-}" ] \ + || fail 'configuration: SDKHARNESS_JAVA_CLASSPATH is unset (the core and httpclient jars built from this checkout, plus their runtime dependencies)' +command -v javac >/dev/null 2>&1 && command -v java >/dev/null 2>&1 \ + || { result SKIP 'missing-runtime -- no JDK (javac and java) on PATH'; exit 0; } + +# Only the simulator's fixed credential (in Support.java) reaches the SDK: every ambient B2_* is dropped. +for name in $(compgen -e); do case "$name" in B2_*) unset "$name" ;; esac; done + +work="$(mktemp -d "${TMPDIR:-/tmp}/sdkharness-java-resilience.XXXXXX")" || fail 'setup: could not create a scratch directory' +watchdog="" +cleanup() { + [ -z "$watchdog" ] || { pkill -P "$watchdog" 2>/dev/null; kill "$watchdog" 2>/dev/null; wait "$watchdog" 2>/dev/null; } + rm -rf "$work" +} +trap cleanup EXIT + +if ! javac --release 11 -nowarn -d "$work/classes" -cp "$SDKHARNESS_JAVA_CLASSPATH" \ + "$HERE/resilience/Support.java" "$HERE/resilience/$class.java" >"$work/javac.log" 2>&1; then + fail "build: the check did not compile against this checkout: $(tidy "$(head -c 400 "$work/javac.log")")" +fi + +log="$work/check.log" +java -cp "$work/classes:$SDKHARNESS_JAVA_CLASSPATH" "$class" >"$log" 2>&1 & +check_pid=$! +# The watchdog is a background shell that kills the check (and records that it did) after the limit. +( + sleep "$WATCHDOG_SECONDS" + : >"$work/timed-out" + kill -TERM "$check_pid" 2>/dev/null + sleep 2 + kill -KILL "$check_pid" 2>/dev/null +) >/dev/null 2>&1 & +watchdog=$! +if wait "$check_pid"; then rc=0; else rc=$?; fi + +# The check's own output is evidence: pass everything but the result line through. +prefix="RESILIENCE $SLUG $SCENARIO @" +grep -Fv "$prefix" "$log" || true +if [ -e "$work/timed-out" ]; then + fail "timeout: the check was still running after ${WATCHDOG_SECONDS} s and was killed" +fi +count="$(grep -Fc "$prefix" "$log" 2>/dev/null || true)" +[ "$count" = 1 ] || fail "contract: expected one result line, got $count (exit $rc)" +verdict="$(grep -F "$prefix" "$log")"; verdict="${verdict#*: }" +case "$verdict" in + PASS) [ "$rc" = 0 ] || fail "contract: PASS exited $rc"; result PASS - ;; + 'FAIL ('*')') reason="${verdict#FAIL (}"; result FAIL "${reason%)}"; exit 1 ;; + *) fail 'contract: malformed result line' ;; +esac From 555a8eb8f103604a35b06937313b72d60eab58dd Mon Sep 17 00:00:00 2001 From: scarreras Date: Wed, 14 Oct 2026 13:21:16 -0300 Subject: [PATCH 2/2] Apply the conformance review to the resilience checks: fail on cleanup errors, enforce loopback at request time, anchor the verdict match - A bucket or file the check could not delete, or faults it could not clear, now fails the verdict; every step is still attempted and the single result line is still printed. - Every check builds its client from one guarded HTTP client factory (the SDK's request settings, including the per-check socket timeout) that refuses any request whose target host is not 127.0.0.1 and fails the verdict if one was attempted. The control-listener requests are held to the same host. GuardSelfTest proves it. - run-resilience matches the result line at column 1 with index(), not anywhere in a line. - README: say what is guaranteed (formatting and capping, not redaction). Co-Authored-By: Claude Sonnet 5.5 --- .sdkharness/README.md | 22 ++-- .../tests/resilience/GuardSelfTest.java | 86 ++++++++++++++ .sdkharness/tests/resilience/Support.java | 112 ++++++++++++++++-- .sdkharness/tests/run-resilience | 7 +- 4 files changed, 207 insertions(+), 20 deletions(-) create mode 100644 .sdkharness/tests/resilience/GuardSelfTest.java diff --git a/.sdkharness/README.md b/.sdkharness/README.md index 4e9210878..fe6b78d92 100644 --- a/.sdkharness/README.md +++ b/.sdkharness/README.md @@ -53,7 +53,8 @@ error reaches the caller) and the recovery path, read from the simulator's reque Each file cites the B2 pages it asserts. They share `tests/resilience/Support.java`; as for conformance, the wrapper compiles `Support.java` and the check with `javac` against the harness-built jars, then runs the check class. Each check creates its own `sdkharness-res-*` -bucket, clears every fault it armed, and deletes everything it made, on failure too. +bucket, clears every fault it armed, and tries to delete everything it made, on failure too; it attempts +every step, and one that fails turns the verdict into a `FAIL` (`cleanup -- N step(s) failed`). | Scenario | Fault, and what it asserts | |---|---| @@ -74,17 +75,21 @@ Timing is a lower bound only, never an upper bound (a busy runner would make tha that is still running after 60 s is killed and reported as a failure: the slowest scenario takes about 3 s, so a minute means the SDK never gave up on something. -A failing scenario prints one `SDKHARNESS_RESULT ... FAIL` line with the step and the reason. +A failing resilience scenario prints one `SDKHARNESS_RESULT ... FAIL` line with the step and the reason, +formatted and capped as described under Conformance (formatting, not redaction). ## What the check can and cannot reach - **Only the loopback simulator.** `run-health`, `run-conformance` and `run-resilience` refuse any - `SDKHARNESS_SIMULATOR_URL` (and, for resilience, `SDKHARNESS_SIMULATOR_CONTROL_URL`) that is not `http://127.0.0.1:`. In conformance the SDK is also - held to it at request time: every check builds its client from one guarded HTTP client factory - (`Support.guardedFactory`, plain HTTP only) that refuses any request whose target host is not - `127.0.0.1`, redirect hops included, and a refused request fails the verdict + `SDKHARNESS_SIMULATOR_URL` (and, for resilience, `SDKHARNESS_SIMULATOR_CONTROL_URL`) that is not + `http://127.0.0.1:`. In conformance and resilience the SDK is also held to it at request time: + every check builds its client from one guarded HTTP client factory (`Support.guardedFactory`, plain + HTTP only; in resilience it carries the SDK's request settings including the per-check socket + timeout, and the control-listener requests are held to the same host) that refuses any request whose + target host is not `127.0.0.1`, redirect hops included, and a refused request fails the verdict (`loopback guard -- blocked a request to non-loopback host ...`). `health` is guarded by its origin - check only. `conformance/GuardSelfTest.java` proves the guard (see below). + check only. `conformance/GuardSelfTest.java` and `resilience/GuardSelfTest.java` prove the guard (see + below). - **Only the simulator's fixed test credential** (`test-key-id` / `test-key`, in the source). Every ambient `B2_*` variable except `B2_BUCKET_NAME` is dropped first (conformance and resilience drop all of them and make their own buckets). @@ -98,7 +103,8 @@ A failing scenario prints one `SDKHARNESS_RESULT ... FAIL` line with the step an For conformance, set `SDKHARNESS_TEST_LEVEL=conformance`, `SDKHARNESS_SCENARIO=` and run `.sdkharness/tests/run-conformance` (needs `javac`, and no `B2_BUCKET_NAME`). -To re-prove the loopback guard (no simulator needed; `$CP` is the same class path): +To re-prove the loopback guard (no simulator needed; `$CP` is the same class path; use `resilience` +in place of `conformance` for the resilience copy): ```bash javac --release 11 -d /tmp/guard -cp "$CP" .sdkharness/tests/conformance/Support.java .sdkharness/tests/conformance/GuardSelfTest.java diff --git a/.sdkharness/tests/resilience/GuardSelfTest.java b/.sdkharness/tests/resilience/GuardSelfTest.java new file mode 100644 index 000000000..fc0a35826 --- /dev/null +++ b/.sdkharness/tests/resilience/GuardSelfTest.java @@ -0,0 +1,86 @@ +// Copyright 2026, Backblaze Inc. All Rights Reserved. +// License https://www.backblaze.com/using_b2_code.html + +import com.backblaze.b2.client.webApiHttpClient.HttpClientFactory; +import com.backblaze.b2.client.webApiHttpClient.HttpClientFactoryImpl; +import com.sun.net.httpserver.HttpServer; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; + +import java.net.InetAddress; +import java.net.InetSocketAddress; + +/** + * Proves the loopback guard in Support. Not a harness scenario (it has no row in tests.tsv); run by + * hand, see ../../README.md. Needs no simulator: it starts its own 127.0.0.1 server. + * + *

It checks that a request to a loopback server passes, that an absolute request URI to a + * non-loopback host (a documentation address, never contacted) is refused before it is sent, and that + * a loopback server redirecting to a non-loopback host is refused on the redirect hop. + */ +public final class GuardSelfTest { + private GuardSelfTest() { + } + + public static void main(String[] args) throws Exception { + final HttpServer server = HttpServer.create(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0), 0); + server.createContext("/ok", ex -> { + ex.sendResponseHeaders(200, -1); + ex.close(); + }); + server.createContext("/redirect", ex -> { + ex.getResponseHeaders().add("Location", "http://203.0.113.7:9/elsewhere"); + ex.sendResponseHeaders(302, -1); + ex.close(); + }); + server.start(); + final String base = "http://127.0.0.1:" + server.getAddress().getPort(); + final HttpClientFactory factory = Support.guardedFactory( + HttpClientFactoryImpl.builder().setSupportInsecureHttp(true).createRequestConfig()); + String problem = null; + try (CloseableHttpClient client = factory.create()) { + try (CloseableHttpResponse r = client.execute(new HttpGet(base + "/ok"))) { + if (r.getStatusLine().getStatusCode() != 200) { + problem = "a loopback request did not reach the server"; + } + } + if (problem == null && !Support.blocked.isEmpty()) { + problem = "a loopback request was blocked"; + } + if (problem == null) { + problem = mustBeBlocked(client, new HttpGet("http://203.0.113.7:9/absolute"), "an absolute non-loopback URL", "203.0.113.7"); + } + if (problem == null) { + problem = mustBeBlocked(client, new HttpGet(base + "/redirect"), "a redirect to a non-loopback host", "203.0.113.7"); + } + if (problem == null) { + problem = mustBeBlocked(client, new HttpGet("http://localhost:" + server.getAddress().getPort() + "/ok"), + "a loopback name that is not 127.0.0.1", "localhost"); + } + } catch (Exception e) { + problem = "unexpected " + Support.describe(e); + } finally { + factory.close(); + server.stop(0); + } + if (problem == null) { + System.out.println("GUARD-SELFTEST " + Support.SLUG + ": PASS"); + } else { + System.out.println("GUARD-SELFTEST " + Support.SLUG + ": FAIL (" + Support.tidy(problem) + ")"); + System.exit(1); + } + } + + private static String mustBeBlocked(CloseableHttpClient client, HttpGet get, String what, String host) { + final int before = Support.blocked.size(); + try (CloseableHttpResponse r = client.execute(get)) { + return what + " was sent (status " + r.getStatusLine().getStatusCode() + ")"; + } catch (Exception e) { + if (Support.blocked.size() != before + 1 || !host.equals(Support.blocked.get(before))) { + return what + " failed without being blocked by the guard: " + Support.describe(e); + } + return null; + } + } +} diff --git a/.sdkharness/tests/resilience/Support.java b/.sdkharness/tests/resilience/Support.java index f57d03654..a04f2bbb3 100644 --- a/.sdkharness/tests/resilience/Support.java +++ b/.sdkharness/tests/resilience/Support.java @@ -12,7 +12,20 @@ import com.backblaze.b2.client.structures.B2ListFileVersionsRequest; import com.backblaze.b2.client.structures.B2UploadFileRequest; import com.backblaze.b2.client.webApiHttpClient.B2StorageHttpClientBuilder; +import com.backblaze.b2.client.webApiHttpClient.HttpClientFactory; import com.backblaze.b2.client.webApiHttpClient.HttpClientFactoryImpl; +import org.apache.http.HttpException; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.protocol.HttpCoreContext; import java.net.URI; import java.net.http.HttpClient; @@ -21,6 +34,7 @@ import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Random; import java.util.UUID; @@ -39,8 +53,14 @@ * journal ({@code GET /journal}: sequence, endpoint, status, fault, an opaque upload URL id; never a * header, token or body). {@link #run} prints exactly one result line, * {@code RESILIENCE b2-sdk-java @simulator: PASS | FAIL ( -- )}, after it - * has cleared every fault and deleted every bucket and file the check created. Anything else it - * prints is a {@code NOTE} line. It never prints a response body or a credential. + * has cleared every fault and tried to delete every bucket and file the check created; a step that + * fails turns the verdict into a FAIL. Anything else it prints is a {@code NOTE} line. Its own lines + * are one capped printable line ({@link #tidy}) built from exception class names, status codes and the + * check's own assertions, never from a response body; this is formatting, not redaction (the only + * credential is the simulator's fixed test one). + * + *

Every SDK request goes through {@link #guardedFactory}, which refuses any request whose target + * host is not 127.0.0.1 and fails the verdict if one was attempted. {@code GuardSelfTest} proves it. */ public final class Support { static final String SLUG = "b2-sdk-java"; @@ -100,6 +120,10 @@ public String toString() { private static String step = "configuration"; private static final List created = new ArrayList<>(); private static int journalMark; + private static final String LOOPBACK_HOST = "127.0.0.1"; + /** Hosts of requests the guard refused (the SDK may swallow the exception, so the verdict reads this). */ + static final List blocked = Collections.synchronizedList(new ArrayList<>()); + private static final List cleanupErrors = new ArrayList<>(); private Support() { } @@ -122,7 +146,7 @@ static void run(String id, int socketTimeoutSeconds, Body body) { try { final String url = System.getenv("SDKHARNESS_SIMULATOR_URL"); control = System.getenv("SDKHARNESS_SIMULATOR_CONTROL_URL"); - // run-resilience checked both already; the SDK's http switch must never reach anything else. + // run-resilience checked both already; the guard in newClient enforces it again on every SDK request. if (url == null || !url.matches(LOOPBACK) || control == null || !control.matches(LOOPBACK)) { throw new Failure("the simulator and control URLs must be IPv4 loopback HTTP origins"); } @@ -136,6 +160,11 @@ static void run(String id, int socketTimeoutSeconds, Body body) { } finally { cleanup(); } + if (!blocked.isEmpty()) { + failure = "loopback guard -- blocked a request to non-loopback host " + tidy(blocked.get(0)); + } else if (failure == null && !cleanupErrors.isEmpty()) { + failure = "cleanup -- " + cleanupErrors.size() + " step(s) failed: " + tidy(String.join("; ", cleanupErrors)); + } if (failure == null) { System.out.println("RESILIENCE " + SLUG + " " + scenario + " @simulator: PASS"); } else { @@ -148,13 +177,73 @@ private static B2StorageClient newClient(String masterUrl, int socketTimeoutSeco final B2ClientConfig config = B2ClientConfig.builder(KEY_ID, KEY, "b2-sdk-java-sdkharness-resilience") .setMasterUrl(masterUrl) .build(); - // The simulator speaks plain HTTP on loopback, so this enables the SDK's documented - // test-environment switch; the SDK's own retry policy and every other default stay as shipped. + // The SDK's own request settings (timeouts, no URI normalization), with the one setting a check + // may change; the SDK's own retry policy and every other default stay as shipped. The guarded + // factory below supplies plain HTTP for the simulator, the SDK's documented test-environment switch. final HttpClientFactoryImpl.Builder httpBuilder = HttpClientFactoryImpl.builder().setSupportInsecureHttp(true); if (socketTimeoutSeconds > 0) { httpBuilder.setSocketTimeoutSeconds(socketTimeoutSeconds); } - return B2StorageHttpClientBuilder.builder(config).setHttpClientFactory(httpBuilder.build()).build(); + return B2StorageHttpClientBuilder.builder(config) + .setHttpClientFactory(guardedFactory(httpBuilder.createRequestConfig())).build(); + } + + /** + * The one HTTP client factory every check uses. It mirrors HttpClientFactoryImpl (same request + * config, pooled connections revalidated after 2 s, 4 s keep-alive) with plain HTTP only, and an + * interceptor that refuses any request whose target host is not 127.0.0.1 before it is sent, + * including each hop of a redirect. A refused request is recorded in {@link #blocked}, which fails + * the verdict even if the SDK handles the exception. + */ + static HttpClientFactory guardedFactory(RequestConfig requestConfig) { + final Registry registry = RegistryBuilder.create() + .register("http", new PlainConnectionSocketFactory()).build(); + final PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(registry); + connectionManager.setMaxTotal(100); + connectionManager.setDefaultMaxPerRoute(100); + connectionManager.setValidateAfterInactivity(2000); + return new HttpClientFactory() { + @Override + public CloseableHttpClient create() { + return HttpClients.custom() + .setConnectionManager(connectionManager) + .setConnectionManagerShared(true) + .setDefaultRequestConfig(requestConfig) + .setKeepAliveStrategy((response, context) -> 4000) + .addInterceptorFirst((HttpRequestInterceptor) (request, context) -> + requireLoopback(request, (HttpHost) context.getAttribute(HttpCoreContext.HTTP_TARGET_HOST))) + .build(); + } + + @Override + public void close() { + connectionManager.shutdown(); + } + }; + } + + /** + * Throws, and records the host, unless the request's target is 127.0.0.1. The target host is the + * one the client is about to connect to (set on the context before interceptors run, also on each + * redirect hop); an absolute request URI is checked as well, and a missing target fails closed. + */ + static void requireLoopback(org.apache.http.HttpRequest request, HttpHost target) throws HttpException { + String host = target == null ? null : target.getHostName(); + if (LOOPBACK_HOST.equals(host)) { + final String uri = request.getRequestLine().getUri(); + try { + final URI parsed = URI.create(uri); + if (parsed.isAbsolute()) { + host = parsed.getHost(); + } + } catch (IllegalArgumentException e) { + host = "unparseable-uri"; + } + } + if (!LOOPBACK_HOST.equals(host)) { + blocked.add(String.valueOf(host)); + throw new HttpException("blocked non-loopback host: " + host); + } } /** @@ -167,7 +256,7 @@ private static void cleanup() { send("DELETE", "/faults", null); } } catch (Exception e) { - note("clearing the faults failed: " + describe(e)); + cleanupErrors.add("clearing the faults: " + describe(e)); } if (client == null) { return; @@ -177,7 +266,7 @@ private static void cleanup() { client.deleteAllFilesInBucket(bucket.getBucketId()); client.deleteBucket(bucket.getBucketId()); } catch (B2Exception | RuntimeException e) { - note("cleanup of " + bucket.getBucketName() + " failed: " + describe(e)); + cleanupErrors.add(bucket.getBucketName() + ": " + describe(e)); } } try { @@ -226,7 +315,12 @@ static String describe(Throwable t) { // ---- the simulator's control listener ------------------------------------------------------ private static String send(String method, String path, String json) throws Exception { - final HttpRequest.Builder request = HttpRequest.newBuilder(URI.create(control + path)); + final URI uri = URI.create(control + path); + if (!LOOPBACK_HOST.equals(uri.getHost())) { + blocked.add(String.valueOf(uri.getHost())); + fail("the control request would leave loopback"); + } + final HttpRequest.Builder request = HttpRequest.newBuilder(uri); if (json == null) { request.method(method, HttpRequest.BodyPublishers.noBody()); } else { diff --git a/.sdkharness/tests/run-resilience b/.sdkharness/tests/run-resilience index 95b551f23..328424697 100755 --- a/.sdkharness/tests/run-resilience +++ b/.sdkharness/tests/run-resilience @@ -87,13 +87,14 @@ if wait "$check_pid"; then rc=0; else rc=$?; fi # The check's own output is evidence: pass everything but the result line through. prefix="RESILIENCE $SLUG $SCENARIO @" -grep -Fv "$prefix" "$log" || true +# A result line starts at column 1 (index(), not a regex: the dots in a scenario id are literal). +awk -v p="$prefix" 'index($0, p) != 1' "$log" || true if [ -e "$work/timed-out" ]; then fail "timeout: the check was still running after ${WATCHDOG_SECONDS} s and was killed" fi -count="$(grep -Fc "$prefix" "$log" 2>/dev/null || true)" +count="$(awk -v p="$prefix" 'index($0, p) == 1 { n++ } END { print n + 0 }' "$log")" [ "$count" = 1 ] || fail "contract: expected one result line, got $count (exit $rc)" -verdict="$(grep -F "$prefix" "$log")"; verdict="${verdict#*: }" +verdict="$(awk -v p="$prefix" 'index($0, p) == 1' "$log")"; verdict="${verdict#*: }" case "$verdict" in PASS) [ "$rc" = 0 ] || fail "contract: PASS exited $rc"; result PASS - ;; 'FAIL ('*')') reason="${verdict#FAIL (}"; result FAIL "${reason%)}"; exit 1 ;;