diff --git a/.bot/config.yaml b/.bot/config.yaml index 5fb30d183..56ad6d59d 100644 --- a/.bot/config.yaml +++ b/.bot/config.yaml @@ -1,7 +1,4 @@ -# Engineer-bot task config for databricks-jdbc (the Databricks JDBC driver). -# Loaded by databricks_bot_engine.engineer_bot.bot_config.load_bot. The reviewer -# needs NO .bot/config.yaml (its prompts are engine-owned + the optional additive -# .bot/prompts/review/system.md); only the engineer-bot is config-driven. +# Engineer-bot task config and shared retrospective lesson routing. name: databricks_jdbc_bugfix marker_namespace: engineer-bot-jdbc bot_login_prefix: peco-engineer-bot @@ -87,11 +84,6 @@ author: issue_url: ISSUE_URL context_files: - issue_body.txt # {{issue_body}} - # Close the loop: the author phase reads this log back so fixes benefit from - # what the retrospective has learned. MUST match `retrospective.log_path` below - # — the retrospective WRITES that path; the author only READS it if the two - # agree. (Set together, they connect learning → application.) - knowledge_log: .claude/knowledge/learning-log.md # Engine orchestration for the author phase. `bug-fix` runs the plan → # author_tests → fix pipeline (write a failing test → fix the code → re-run to @@ -101,21 +93,14 @@ author: # prompts/engineer/system.md, not here. flow: bug-fix -# Daily learning extraction (retrospective flow). Run by engineer-bot-learning.yml -# via `python -m databricks_bot_engine.engineer_bot.retrospective`: over an adaptive -# look-back window it sweeps every merged PR (diff + review comments, which the -# engine gathers ITSELF per PR) plus engineer-bot author runs, and if the model -# finds a durable, reusable learning, opens/updates a single rolling PR appending a -# dated section to log_path. Human-gated — never commits the canonical log directly. -# Omitting this block makes the retrospective a no-op. -# -# `system_prompt` is intentionally OMITTED: this repo ships no -# prompts/retrospective_system.md, so the engine's built-in base prompt is used. -# (A set-but-missing system_prompt file would be a hard error; an UNSET key uses -# the base. Add the key later only if you write a repo-specific additive prompt.) -# An initial learning log is seeded at log_path so the author.knowledge_log read -# path is a real file from day one; the retrospective appends dated sections to it. +# Both bots receive matching lessons; the old log is a frozen archive. retrospective: - log_path: .claude/knowledge/learning-log.md # MUST match author.knowledge_log above - branch_prefix: ai/learning-pr- + log_mode: distill + log_path: .claude/knowledge/learning-log.md + journal_dir: .claude/knowledge/journal + lessons_path: .claude/knowledge/lessons.md + auto_merge_learning_pr: false + author_workflow: engineer-bot.yml + # Keep the legacy append-mode PR (#1637) on its original branch. + branch_prefix: ai/learning-distill-pr- pr_label: engineer-bot-learning diff --git a/.claude/knowledge/learning-log.md b/.claude/knowledge/learning-log.md index 988c85f46..1429c29e8 100644 --- a/.claude/knowledge/learning-log.md +++ b/.claude/knowledge/learning-log.md @@ -1,12 +1,267 @@ -# Engineer-bot learning log +# Frozen learning archive — databricks-jdbc -This file is the canonical, human-gated knowledge log shared by two consumers -configured in `.bot/config.yaml`: +Preserves the entries from PR #1637 at +`d6b0766be669405022ed1a60d7a54e950fadfe6e` before migration to distill mode. +New daily entries belong in `journal/`; bots receive distilled rules from +`lessons.md`. This archive is never updated or injected into bot prompts. -- `retrospective.log_path` — the retrospective flow APPENDS a dated section here - when it extracts a durable, reusable learning (via a human-gated rolling PR). -- `author.knowledge_log` — the author phase READS this file so fixes benefit from - what has been learned. Seeded here so the read path is never a missing file. +Lesson counts measure archived occurrences, not distinct PRs. A +`log:YYYY-MM-DD` source points to that dated section below, including its author-run IDs. -No learnings have been recorded yet. Dated sections are appended below by the -retrospective flow. +## Entries + +### 2026-08-14: learnings since 2026-08-13T17:41:38Z +- **Context:** Author run #31744986912 fixed a parameter-bind bug where `DatabricksTypeUtil.getDatabricksTypeFromSQLType` collapsed both `Types.FLOAT` and `Types.REAL` to the 4-byte Databricks `FLOAT` wire type, silently narrowing FLOAT binds. + **Rule:** Per the JDBC spec (Appendix B type table), `Types.FLOAT` is a synonym for `DOUBLE` (8-byte double precision, Java `double`) and must map to the 8-byte wire type; only `Types.REAL` is 4-byte single precision (Java `float`) — never collapse FLOAT and REAL to the same Databricks type, and treat the declared SQL target type (not the bound value's native type) as what drives the wire type. +- **Context:** In author run #31744986912 the read-only diagnosis/plan phase spent a turn calling `edit_file` to create a test file (turn 44), only to discover editing tools are unavailable in that phase (turn 46) — the deliverable is the structured plan, not code. + **Rule:** The diagnosis/plan phase is read-only; don't attempt `edit_file`/write operations there. Defer all file creation and edits to the author_tests/fix phases and keep the diagnosis phase to reads, greps, and the structured plan output. + +### 2026-08-18: learnings since 2026-08-17T17:33:39Z +- **Context:** PR #1582 wired `merge_group` triggers into required-check workflows (checkNextChangelog, releaseFreeze, prCheck) for a GitHub merge queue. + **Rule:** For a required status check to stay valid under a GitHub merge queue, register the workflow on the `merge_group` event; PR-semantic jobs that read `github.event.pull_request.*` should self-skip in the queue with `if: github.event_name != 'merge_group'` — a skipped required check counts as a pass under branch protection, so the queue is not blocked while the gate stays enforced at PR time. +- **Context:** In PR #1582 the engineer-bot re-posted the same "NEEDS HUMAN DECISION / blocked" verdict on successive replies of one review thread because the reviewer's finding required editing `.github/workflows/*.yml`, a non-writable path in this environment. + **Rule:** When a review finding targets a path the environment cannot write (e.g. `.github/workflows/*`), state once that it is agreed-but-not-actionable-here and stop; do not re-analyze or re-post a blocked verdict on each subsequent thread reply. + +### 2026-08-20: learnings since 2026-08-19T17:33:03Z +- **Context:** PR #1629 added telemetry error-classification guardrails to CLAUDE.md and the PR template, covering any error emitted by the JDBC driver. + **Rule:** When adding or changing a driver-emitted error, use `DatabricksDriverErrorCode` (reuse a matching code or add a uniquely-numbered enum value), add a test asserting the emitted error name and numeric code, and record its driver/server/user classification in the maintainers' telemetry taxonomy — never infer the classification from the error name alone. + +### 2026-08-22: learnings since 2026-08-21T17:33:33Z +- **Context:** PR #1652 fixed connections failing when a parameter appeared in both the JDBC URL and the `Properties` object; the root cause was Guava's `ImmutableMap.Builder.build()`, which throws on duplicate keys. The fix inserts properties first, then URL params, and calls `buildKeepingLast()` so URL wins. + **Rule:** When merging config from multiple sources into a Guava `ImmutableMap.Builder`, `build()` throws on duplicate keys — use `buildKeepingLast()` and insert entries in ascending precedence order (lowest-priority source first) so the highest-priority source wins. +- **Context:** PR #1621's new `BatchParameterSet` normalizes JDBC's 1-based parameter indexes to **zero-based** wire ordinals (`cardinal = index - 1`), while the existing SEA/Thrift path (`DatabricksPreparedStatement.setObject` → `mapToParameterListItem.setOrdinal`) forwards the raw **1-based** index. A reviewer flagged the off-by-one; the author confirmed the native-batch backend contract expects zero-based ordinals (first param = 0), whereas the current non-batch backend ignores the ordinal and relies on positional order. + **Rule:** Parameter ordinals diverge by execution path — native batching expects 0-based ordinals while the existing SEA/Thrift path sends 1-based JDBC indexes (tolerated only because that backend uses positional order); when wiring native batching through shared `setOrdinal` plumbing, verify the ordinal base against the backend contract and cover both paths with request-capture tests to avoid an off-by-one. + +### 2026-08-26: learnings since 2026-08-25T17:34:14Z +- **Context:** PR #1663 bumped `httpcore5`/`httpcore5-h2` to 5.4.3 while leaving `httpclient5` at 5.5.2 (tested against the httpcore5 5.3.x branch); a reviewer flagged the version skew and the fix was to bump httpclient5 to 5.6.3 so all three came from one published, mutually-tested dependency set. The original test plan built with `-Dmaven.test.skip=true`. + **Rule:** When bumping one library in a tightly-coupled family (e.g. httpclient5/httpcore5), move the whole set to a single published-compatible combination rather than pinning individual minor versions — a lone minor bump under a dependent tested against a different branch surfaces as a runtime NoSuchMethodError/LinkageError, not a compile error, and must be validated with the HTTP-path/integration tests (never a skip-tests build). +- **Context:** In PR #1641 the Thrift-native metadata error-propagation guard keyed off the connection-level request flag `isThriftNativeMetadataRequested()` rather than the response manifest `resultSet.isThriftNativeMetadataResult()`; because the exception is thrown before any result is available, enabling the opt-in flag silently regressed the object-not-found / ALL-CATALOGS-parse-error → empty-ResultSet fallbacks even against legacy servers that ignore the feature header. + **Rule:** Gate behavior changes for an opt-in feature on whether the feature actually took effect (server/response manifest), not on whether it was merely requested — otherwise enabling the flag changes semantics against servers that don't support it, breaking JDBC contract behavior (e.g. getTables/getColumns returning empty for a missing object). +- **Context:** In PR #1641 the native error-propagation re-throw guard was added to `listTables`/`listColumns` but not to the sibling `listFunctions`/`listPrimaryKeys`/`listImportedKeys`/`listCrossReferences`, which kept swallowing object-not-found/parse errors into empty results — reviewers repeatedly flagged the asymmetry as reading like an oversight. + **Rule:** When changing error-handling (or any behavioral) policy in one of a family of near-identical operations, apply it to all of them or add a comment explaining the deliberate divergence; partial application across parallel metadata paths reads as a bug and yields inconsistent contract behavior. +- **Context:** In PR #1641 `copyThriftNativeMetadataRows` copied server native rows purely positionally (`getObject(1..columnCount)`) and downstream consumers relied on fixed indices (`row.get(0)`, `CROSS_REFERENCE_COLUMNS.indexOf(...)`), with no validation of column count or names against the expected `*_COLUMNS` lists. + **Rule:** When consuming a wire/native result positionally against a server-controlled schema, validate column count (and ideally names) against the expected column list, or map by name — otherwise schema drift silently mis-filters/mislabels rows instead of failing loudly. +- **Context:** In PR #1641 `testThriftNativeFormattingMatchesRawThriftBuilder` compared the `DatabricksResultSet`-taking overload against the `List>`-taking overload, but the native branch simply forwards to that same row builder, so the assertion was tautological and never exercised the real risk (native JDBC-ordered rows normalized differently than a legacy SEA result going through `getRows`+adapters). + **Rule:** A test that compares two code paths which delegate to the same underlying method proves only delegation, not equivalence; to guard a real divergence risk, assert the new path's output against an independently-computed reference (e.g. a genuine legacy-adapter result), not another wrapper over the same builder. + +### 2026-08-27: learnings since 2026-08-26T18:05:01Z +- **Context:** PR #1659 fixed a `NullPointerException` in `DatabricksArray.convertElements` when materializing arrays of nested complex types (ARRAY/MAP/STRUCT) that contain literal `null` elements; the reviewer noted the initial fix added a per-branch null check but missed the STRUCT branch, which still NPE'd while building its error message. + **Rule:** Arrays can legally hold `null` for ANY element type, so when converting/materializing nested complex types (ARRAY, MAP, STRUCT), handle `element == null` once at the top of the loop before type-specific dispatch — don't scatter per-branch null checks that are easy to miss for one type. + +### 2026-09-05: learnings since 2026-09-04T17:26:14Z +- **Context:** PR #1672 gated `UseBoundedSeaApi`/`EnableThriftNativeMetadata` defaults on the server-side `enableSqlExecForJdbc` rollout flag via `resolveFeatureFlag`; a reviewer asked whether defaulting the param to `1` makes the server flag redundant. + **Rule:** When defaulting a client feature-flag param to `1` behind a server rollout flag, resolve in priority order — explicit connection param wins; otherwise if the default is `1` defer to the server flag's value; otherwise return false — so a `1` default enables server-controlled rollout rather than forcing the feature on. +- **Context:** In PR #1672's `NEXT_CHANGELOG.md`, reviewers flagged that a public release note referenced the internal server flag `enableSqlExecForJdbc` and lacked opt-out guidance for a newly default-enabled param. + **Rule:** Public-facing changelog entries for a newly-introduced or default-enabled connection param must describe what the param does and how to opt out, and must not expose internal server-side flag names. + +### 2026-09-08: learnings since 2026-09-07T17:26:57Z +- **Context:** PR #1670 fixed later logging-enabled JDBC connections being unable to emit logs because an earlier `LogLevel=OFF` connection had already flipped the one-time `isLoggerInitialized` static guard (and installed no usable handler). The fix returns from the OFF/no-op path *without* setting the guard, and also leaves it unset when handler installation throws (`testFailedInitializationCanBeRetried`), so a later capable caller can still initialize. + **Rule:** Set a one-time initialization guard (static `isInitialized` flag / singleton) only after a fully-functional configuration is actually installed — never on a disabled/no-op early path or before an operation that can throw — otherwise the first weak or failed caller permanently poisons initialization for every later caller. +- **Context:** In PR #1670 a reviewer flagged that detecting the "logging off" case should compare `level.intValue() == Level.OFF.intValue()` rather than by object reference/equality, since `java.util.logging.Level` allows distinct instances that share the same integer value. + **Rule:** Compare `java.util.logging.Level` (and similar level/enum-like value objects) by `intValue()`, not by `==` reference or `.equals()`, because semantically-equal levels can be different object instances. + +### 2026-09-10: learnings since 2026-09-09T17:27:44Z +- **Context:** PR #1680 fixed `DatabaseMetaData.getTypeInfo()` returning the `INTERVAL` row out of `DATA_TYPE` order; the fix reordered the static row array and added a test asserting rows are sorted by `DATA_TYPE`. + **Rule:** JDBC `DatabaseMetaData` metadata result sets carry spec-mandated ordering contracts (e.g. `getTypeInfo()` MUST return rows ordered by `DATA_TYPE`) — when adding or editing rows in a hardcoded metadata table, preserve the required sort order and guard it with an ordering assertion, since column values alone won't reveal the violation. + +### 2026-09-12: learnings since 2026-09-11T17:27:24Z +- **Context:** In PR #1683 the Reyden auto-recovery double-failure test asserted only on `getMessage()` and `assertNotNull(getCause())`, but the production code preserves the original error via `addSuppressed(e)` while the cause is a *different* exception — so the test would still pass if `addSuppressed` were deleted, leaving the "neither error chain is lost" guarantee untested. + **Rule:** When code preserves an original error as a *suppressed* exception (`addSuppressed`), the test must inspect `getSuppressed()` (and assert its SQLSTATE/identifying message), not just `getCause()`/`getMessage()` — otherwise the suppression is effectively unverified. +- **Context:** PR #1683's `ReydenWarehouseCache` built its `(host, warehouse_id)` cache key with `host.toLowerCase()` using the default JVM locale, which folds `I`/`i` unexpectedly on a Turkish-locale JVM. + **Rule:** Always pass `Locale.ROOT` to `String.toLowerCase()`/`toUpperCase()` when normalizing identifiers, hostnames, or cache keys — locale-default case folding is a latent multi-locale bug. +- **Context:** PR #1683's cache read `System.currentTimeMillis()` directly against a hard-coded 6h TTL, leaving the entire expiry/eviction path untestable without reflection or real sleeps; the fix injected a `LongSupplier` clock and TTL override via a package-private constructor. + **Rule:** Make the time source (and TTL) injectable for any TTL/expiry/eviction logic so the expiry branches can be driven deterministically in tests — code that reads the wall clock directly leaves its core time-dependent behavior unverified. +- **Context:** In PR #1683 the fallback recovery only re-attached the original Thrift `KP001` as suppressed when the SEA fallback threw `DatabricksSQLException`; an unchecked exception (SDK `RuntimeException`/NPE) from the fallback path would escape and drop the original error context entirely. + **Rule:** When a fallback/recovery path wraps its failure to preserve the original error, catch broadly enough (or wrap) to also cover unchecked exceptions from the fallback — a narrow typed catch silently loses the original error on unexpected failure modes. +- **Context:** PR #1683's Reyden auto-recovery silently switched connections that had forced Thrift-only metadata (`UseQueryForMetadata=0` / `TreatMetadataCatalogNameAsPattern=1`) over to SEA, which serves metadata differently; the reviewer had it emit a `LOGGER.warn` so the behavior change is observable rather than a debug-only line. + **Rule:** When automatic recovery/fallback overrides an explicit user configuration, log it at WARN so the otherwise-silent behavior change is observable — don't bury a semantics change at debug level. + +### 2026-09-17: learnings since 2026-09-16T17:27:33Z +- **Context:** In PR #1678 (session-version tracking in `DatabricksSession`), the reviewer flagged mixing an `AtomicReference` CAS loop *inside* a `synchronized (this)` block while a single out-of-monitor `set(null)` was the only thing the atomic was load-bearing for; the fix collapsed to a plain `volatile` nullable field. + **Rule:** Don't mix concurrency mechanisms on one field — if every mutation is already guarded by the monitor, a CAS/AtomicReference is redundant; use a monitor-guarded plain field plus `volatile` for lock-free reads, or go fully lock-free, but not both. +- **Context:** In PR #1678 the reviewer caught `execution_mode=FAST` being set unconditionally on every SEA `CreateSessionRequest`, even though the changelog and PR scoped the feature to Lakehouse Real-Time warehouses, with no feature flag or warehouse-type gate. + **Rule:** A request/protocol change motivated by one feature but applied to all sessions/connections can silently alter behavior for users who never opted in — gate feature-scoped request fields behind a flag or type detection, and document the wire-level change in the changelog. + +### 2026-10-01: learnings since 2026-09-30T17:30:42Z +- **Context:** PR #1656 bumped `httpclient5` 5.6.3→5.6.4; the review bot flagged that a bare dependency bump needs either a `NEXT_CHANGELOG.md` entry under `### Updated` or a `NO_CHANGELOG=true` marker in the PR description. + **Rule:** Dependency bumps are user-visible: always add a `NEXT_CHANGELOG.md` entry (or set `NO_CHANGELOG=true`), or the `check-next-changelog` CI gate fails. +- **Context:** PR #1703 fixed a failing M2M replay test whose WireMock mapping used an exact `equalTo` body pattern for the `/api/2.0/sql/sessions/` POST; the client now sends an added `execution_mode":"FAST"` field, so the stored session-request JSON had to be regenerated to match. + **Rule:** WireMock `equalTo` body matchers are exact — any new field in a client request payload (e.g. session-creation params) breaks replay tests until every stored mapping JSON is updated in lockstep. + +### 2026-10-03: learnings since 2026-10-02T17:30:45Z +- **Context:** PR #1709 review flagged `UserAgentManager`/`ClientConfigurator` tests that call `UserAgent.withOtherInfo`/`withProduct` in `@BeforeAll` with no teardown. + **Rule:** The SDK's `UserAgent` is a process-wide static accumulator — mutating it in tests without snapshot/restore leaks segments into sibling tests and causes order-dependent flakiness; always restore it (or avoid global mutation) in UserAgent-related tests. +- **Context:** PR #1709 reviewer repeatedly (3x) flagged an "unguarded `UserAgent.sanitize()` throws on the SEA request-path lambda" bug in `orderSeaUserAgent`; the author showed it was a false alarm. + **Rule:** `UserAgent.sanitize()` only applies a fixed regex replacement and cannot throw `IllegalArgumentException` — the exception the sibling call sites catch comes from `withOtherInfo`'s key validation, so a bare `sanitize` call (e.g. inside the request-path transport wrapper) needs no try/catch. + +### 2026-10-05: learnings since 2026-10-04T18:11:23Z +- **Context:** PR #1707 unified OAuth `Auth_Scope` parsing but a reviewer flagged that the sibling auth flows had drifted — client-secret M2M and U2M split the value on whitespace into multiple scopes, while JWT-assertion M2M still wrapped the raw string in `Collections.singletonList(...)`, so `Auth_Scope=sql jobs` became one malformed scope `"sql jobs"` on that path. + **Rule:** When changing how a connection/config parameter is parsed in one OAuth/auth flow, audit all sibling flows (U2M, client-secret M2M, JWT-assertion M2M) and route them through a shared helper, since per-flow copies silently drift in intent. +- **Context:** In PR #1707 a reviewer caught that the new M2M scope guard checked only `authScope != null`; a URL like `Auth_Scope=` yields `""` (not null), so the code set scopes to `[""]`, overriding the SDK's `all-apis` default with an empty scope and silently failing token acquisition instead of falling back. + **Rule:** When a blank connection/config value can override a non-empty library default, guard on blank/empty (not just null) so a misconfiguration degrades to the default rather than a confusing downstream auth failure. +- **Context:** In PR #1707 the client-secret M2M tests asserted only `config.getScopes()` (the config field), whereas the JWT-assertion tests captured the outbound HTTP token request and parsed its `scope` param via `URLEncodedUtils`; the config-field assertion cannot catch the SDK silently ignoring or mistranslating the field on the wire. + **Rule:** When behavior depends on a value a downstream SDK is supposed to forward, assert on the actual outbound request (capture the HTTP request/params), not just that the config field was set. + +### 2026-10-07: learnings since 2026-10-06T17:29:45Z +- **Context:** PR #1723 made SQL Execution API (SEA) requests send `Accept-Encoding: identity` by default (new `EnableSeaResponseCompression`, default off), because SEA inline Arrow results are already LZ4-compressed, so HTTP gzip just re-compresses already-compressed bytes for no gain. + **Rule:** Don't enable HTTP response compression (gzip) on responses whose payload is already compressed (e.g. SEA inline LZ4-Arrow results) — it burns CPU without shrinking bytes; default such transports to `Accept-Encoding: identity` and gate re-compression behind an opt-in flag for bandwidth-constrained links. +- **Context:** In PR #1713 a reviewer flagged that the new typed feature-flag getters (`getInt32`/`getInt64`/`getDouble`/`getString`/`getStringList`) all route through a `parse()` that null-guards the key and returns empty, while the sibling `getBoolean(name)` calls Guava `Cache.getIfPresent(null)` directly and throws NPE on a null key — an inconsistent contract across a method family meant to be used uniformly. + **Rule:** When adding a family of related accessors, keep their null/missing/invalid-input handling uniform (all throw or all return empty/Optional) — and note that Guava `Cache.getIfPresent(null)` rejects null keys with NPE, so route keys through a shared guard if the family's contract is "missing → empty". + +### 2026-10-14: learnings since 2026-10-13T17:32:05Z +- **Context:** PR #1726 fixed per-connection latency and lock contention in long-running JVMs: the SDK's `UserAgent.withOtherInfo` appends unconditionally on every call and `UserAgent.asString` re-formats the entire accumulated list on every request, so calling it per-connection made the shared User-Agent grow without bound. + **Rule:** The SDK `UserAgent` registry is append-only and process-global (`withOtherInfo` never dedupes, `asString` reformats the whole list per request); register each distinct entry at most once per JVM and bound the total — never add entries on a per-connection/per-request path. +- **Context:** In PR #1726 the reviewer noted that once the JVM-global User-Agent entry cap is hit, SEA (SQL Execution API) requests still carry the customer entry because `orderSeaUserAgent` re-injects it per request, but Thrift requests rely solely on the shared list and silently lose entries past the cap. + **Rule:** SEA and Thrift paths source the User-Agent differently (SEA re-injects per request, Thrift reads only the shared global list); when changing shared User-Agent registration, verify both transports still carry the expected entries rather than assuming parity. + +### 2026-10-16: learnings since 2026-10-15T17:29:37Z +- **Context:** PR #1728 fixed queries throwing `Multiple entries with same key` when result column names differ only in case (e.g. `policyid` vs `policyId`). The old code normalized keys to lowercase and collected via Guava's `ImmutableMap.toImmutableMap`, which throws on any post-normalization key collision; the fix switched to a `putIfAbsent`/first-wins builder applied consistently across every SEA and Thrift `DatabricksResultSetMetaData` constructor. + **Rule:** When building a case-insensitive column-name→index lookup, dedupe with first-wins (`putIfAbsent`) semantics rather than `ImmutableMap.toImmutableMap` (which throws on keys colliding only after lowercasing); the JDBC spec requires label lookups to return the first matching column, and the behavior must be identical across all SEA and Thrift metadata paths. + +### 2026-10-19: learnings since 2026-10-18T17:27:33Z +- **Context:** In PR #1700 a new paren-depth-aware `parseMapMetadata` correctly split `MAP`, but every consumer re-split the re-joined string with `.split(",", 2)`, breaking on the comma inside `DECIMAL(10,2)`; a separate finding showed struct field names like `` STRUCT<`a,b`:INT> `` are split inside the quoted name. + **Rule:** Type-metadata splitting (map key/value, struct fields) must be depth- and quote-aware, and all consumers must consume the structured parts (e.g. a `*Parts` accessor) — never re-split a flattened/re-joined metadata string on a naive delimiter. +- **Context:** In PR #1700 the `MetadataParser`-level unit test for the new paren-aware map split passed and gave false confidence, because it never exercised the consumer re-split path (`parseToMap`/`formatMapString`/`DatabricksMap`) where the bug actually lived. + **Rule:** A green helper/parser-level unit test does not prove the feature works end-to-end; add a regression that exercises the real caller/call-path, not just the helper in isolation. +- **Context:** In PR #1700 a new native-Arrow constructor defensively copied the incoming WKB (`Arrays.copyOf`), but `getWKB()` still returned the internal array reference, so a caller could mutate stored state in place — undermining the input copy. + **Rule:** Defensive copying of mutable fields (byte[]/arrays/collections) must be symmetric: copy on both ingest (constructor/setter) and egress (getter); copying only one side still leaks mutable internal state. +- **Context:** In PR #1700 the top-level path special-cased `SRID==0` to emit bare WKT, while nested struct/array/map values used `toString()` which unconditionally emitted `SRID=0;...`, so the same logical value serialized differently depending on nesting depth and on the geospatial-support flag. + **Rule:** When a textual/serialized representation has special cases (e.g. omit-when-zero), route top-level, nested, and feature-flag variants through one shared formatter so the representation is identical regardless of nesting depth or flag state. +- **Context:** In PR #1700 the complex-support-disabled string fallback decoded and re-serialized the whole container to transform one geospatial leaf; this coerced unrelated siblings (DECIMAL lost precision, DATE/TIMESTAMP lost wire form, BINARY became `[B@...` JVM identity) and emitted unquoted geospatial text, yielding invalid JSON from `getString()`. + **Rule:** When reserializing a container to transform one leaf type, preserve the exact serialization of all other fields and JSON-quote/escape string leaves; do not round every value through typed JDBC objects or rely on a domain object's `toString()` for container output. +- **Context:** In PR #1700 the disabled-complex geospatial path parsed containers before returning a string; for `MAP>` with a null value `DatabricksMap.convertValue` called `getClass()`/switched on the null, throwing instead of returning the result. + **Rule:** In container value converters, short-circuit null before any type-specific dispatch that calls `getClass()`/`switch`/`instanceof` on the value, so null elements/map-values do not crash conversion. +- **Context:** In PR #1700 a WKB-to-WKT decode path moved from test-only to production, but its empty-geometry tests were seeded via JTS's own `toWKB` writer rather than raw server wire bytes, so they never exercised how the server actually encodes edge cases (e.g. `POINT EMPTY` with NaN coordinates) that `WKBReader` is historically finicky about. + **Rule:** Test a decode/parse path with raw wire bytes captured from the real producer, not data round-tripped through the same library's writer — writer round-trips mask serialization quirks the real source emits. + +### 2026-10-20: learnings since 2026-10-19T17:29:57Z +- **Context:** PR #1731 added bounded-SEA inline Arrow and reworked `DatabricksResultSet` cursor logic (`isAfterLast`/`isLast`/`hasUpdateCount`/`getTotalRows`) plus `DatabricksResultSetMetaData` to stop relying on the manifest `total_row_count`. + **Rule:** SEA/bounded-SEA Arrow result manifests may omit `total_row_count` — never use it directly for cursor position or update-count logic; drive continuation from `next_chunk_index`, track stream exhaustion in the provider, and gate row-count assumptions behind an explicit `hasKnownTotalRows()`/`-1` sentinel check. +- **Context:** In PR #1731 a reviewer caught that when Arrow is disabled, `useCloudFetchForResult()` returns false, which made the new `boundedSeaInline` branch true and incorrectly requested Arrow — a routing bug in the client-type/param selection logic. + **Rule:** When adding a routing branch gated on a combination of feature flags (CloudFetch / SEA / Arrow), audit the entire parameter waterfall for every flag combination — a flag that is false for an unrelated reason can silently satisfy a downstream condition and misroute; enumerate the flag matrix explicitly rather than reasoning about one flag at a time. + +### 2026-10-22: learnings since 2026-10-21T17:30:16Z +- **Context:** In author run #39734137201, the agent tried `WebFetch`/`WebSearch` to read PR #1466 and search for context (turns 21, 32), then discovered at turn 35 that "Web tools aren't available" in the author-run environment, wasting turns. + **Rule:** Web tools (WebFetch/WebSearch) are NOT available in the engineer-bot author-run environment; diagnose from the local repo instead of attempting to fetch external PRs/URLs/docs. +- **Context:** In author run #39734137201 (bug #1729, NPE on inline URL_BASED_SET results), the TDD repro test for `SHOW SCHEMAS LIKE 'x'` PASSED instead of failing because the provided live connection was a SQL warehouse, while the bug only reproduces on an all-purpose cluster over Thrift protocolv1 that returns URL_BASED_SET with inline data. + **Rule:** A server-response-format-dependent bug only reproduces on the cluster type that emits that format (e.g. all-purpose cluster / Thrift protocolv1 vs SQL warehouse); verify the e2e connection can actually produce the failure mode before trusting a green repro test — a passing repro may mean wrong environment, not absence of the bug. +- **Context:** Author run #39734137201 diagnosed that for a Thrift connection with cloud fetch enabled, small queries return result format URL_BASED_SET but carry data inline in `TRowSet.arrowBatches`, leaving `resultData.getResultLinks()` null; code in `AbstractRemoteChunkProvider.populateChunkIndexMap` iterated the null links and NPE'd. A bare null-guard would wrongly yield an empty result set. + **Rule:** URL_BASED_SET responses can carry data inline (arrowBatches) with null result links — code consuming result links must detect this case and route to the inline-Arrow handler, not just null-guard (which silently drops rows). +- **Context:** In author run #39813458379, complex-type e2e tests in ComplexTypeQueryTests were added parameterized over both SEA (usethriftclient=0) and Thrift (usethriftclient=1) with EnableComplexDatatypeSupport=1. + **Rule:** Complex-type (ARRAY/MAP/STRUCT) e2e coverage must be parameterized across both the SEA and Thrift clients (and set EnableComplexDatatypeSupport=1), since the two client paths diverge and a single-client test leaves the other path unverified. +- **Context:** In author run #39813458379, the implement phase re-read the same files the plan/diagnosis phase had already examined (DatabricksResultSet constructors, ColumnMetadata, DatabricksTypeUtil, MetadataParser, the e2e test file), repeating most of the exploration across the phase boundary. + **Rule:** Diagnosis-phase findings (exact file:line anchors, constructor signatures, helper locations) should be carried into the implement-phase prompt so the implement agent doesn't re-grep/re-read the same files from scratch. + +### 2026-10-23: learnings since 2026-10-22T17:30:18Z +- **Context:** PR #1734 fixed `ArrowStreamResult` so bounded SEA inline Arrow results use `SeaInlineArrowChunkProvider` even when CloudFetch (`EnableQueryResultDownload=1`) is enabled. The old check also required `!isCloudFetchEnabled()`, so hybrid inline results (an `attachment` plus a `nextChunkIndex` pointing at more inline chunks) were sent to the wrong provider. + **Rule:** Pick the SEA result/chunk provider from the shape of the server response (e.g. `ResultData.attachment != null`), not from client config flags like CloudFetch. The server can return inline Arrow even when CloudFetch is enabled, so add a test for the hybrid inline + next-chunk case. +- **Context:** PR #1734 also fixed `ArrowStreamResult.next()`. It incremented `currentRowIndex` before advancing, and it ignored the boolean that `chunkProvider.next()` returns, so the cursor state was wrong once chunks ran out. + **Rule:** In ResultSet/chunk iterators, check the return value of the advance call (`chunkProvider.next()`) first, and only update cursor state (row index) after the advance succeeds. Return false when advancing fails, rather than fetching a chunk that doesn't exist. + +### 2026-10-27: learnings since 2026-10-26T17:31:04Z +- **Context:** In run #40422688812, the author's live e2e DeepCloneGetColumnsTests called `getMetaData().getColumns(null, schema, table, null)` and `getColumns(null, null, table, null)`. With a null catalog the driver scans every catalog on the warehouse, so the run hung until the 20-minute timeout. The author then had to remove those calls and run the test again. + **Rule:** In live/e2e metadata tests, always pass an explicit catalog (and preferably schema) to getColumns/getTables/etc. A null or wildcard catalog scans the whole warehouse and can exceed the 20-minute test timeout. Cover null-catalog behavior with unit tests, not live calls. +- **Context:** PR #1747 removed the `UseQueryForMetadata` param, which used to affect client selection in `getClientTypeFromContext`. The reviewer found that a `@CsvSource` row in `testAutomaticClientSelectionGrid` still appended `;UseQueryForMetadata=0` and expected THRIFT, so it would fail in CI. The author had skipped it on the assumption that only Mockito tests were affected. Removing the `DatabricksJdbcUrlParams` enum entry also makes the key a silently ignored unknown param, so URLs that relied on it as a documented escape hatch change routing with no runtime signal. + **Rule:** When you remove or repurpose a `DatabricksJdbcUrlParams` entry, grep tests for the raw key string, including URL suffixes in parameterized/`@CsvSource` rows, and run the full `DatabricksConnectionContextTest`. Also state the behavior change for users who still set the key in NEXT_CHANGELOG, because removed keys are silently ignored. + +### 2026-10-28: learnings since 2026-10-27T17:26:03Z +- **Context:** PR #1746 added Real-time session routing for SQL Execution API (SEA) sessions. A routing flag (`realtimeSession`, set from the `x-databricks-rt-session` CreateSession response header) had to go through `SessionInfo`, `SessionId` (constructor, `toString`/`deserialize`, `equals`, `getSessionInfo`) and `StatementId.forCurrentSession`, so that later statement, status, cancel and close calls build warehouse-scoped URLs. The serialized form only gained an optional suffix (`s|warehouseId|session-id[|r]`), so existing 3-part IDs still parse. + **Rule:** When you add per-session routing state to SEA identifiers, carry it through every path: SessionInfo, SessionId serialize/deserialize/equals/getSessionInfo, and StatementId. Extend the serialized ID format only with optional trailing parts so older IDs still deserialize, and add a round-trip test. If you miss a path, a deserialized session or statement silently falls back to the default (non-warehouse) URLs. + +### 2026-10-29: learnings since 2026-10-28T17:25:14Z +- **Context:** PR #1739 (`DatabricksArray.getResultSet(long index, int count)`). The first version computed `(int) index - 1` and `start + count` in `int`. The common "everything from here" call `getResultSet(2, Integer.MAX_VALUE)` overflowed and returned an empty result, and `index = 2^32` cast down to a negative start, which threw `ArrayIndexOutOfBoundsException`. The same cast pattern is still in `getArray(long, int)`. + **Rule:** In JDBC windowing APIs that take a `long` index/position and an `int` count (Array.getArray/getResultSet, Blob.getBytes, Clob.getSubString), do the start/end math in `long`, clamp both to the collection length before casting to `int`, and test with `count = Integer.MAX_VALUE` and an index past the end, including one above `Integer.MAX_VALUE`. +- **Context:** PR #1739. A bad caller argument (`index < 1`) first threw `DatabricksDriverException`, which is a RuntimeException, using the conversion error code `COMPLEX_DATA_TYPE_ARRAY_CONVERSION_ERROR`. The reviewer pointed out that the JDBC contract requires a `SQLException`. The fix throws `DatabricksSQLException` with `DatabricksDriverErrorCode.INPUT_VALIDATION_ERROR`, and a test asserts both the SQLState and the vendor error code. + **Rule:** In a JDBC method that declares `throws SQLException`, an invalid caller argument must throw `DatabricksSQLException` with `INPUT_VALIDATION_ERROR`, never an unchecked `DatabricksDriverException` or an unrelated conversion code. Add a test that asserts `getSQLState()` and `getErrorCode()` on that path. +- **Context:** PR #1739 built an in-memory `DatabricksResultSet` whose column types came from complex-type element text. Raw type text such as `DECIMAL(10,2)` or `ARRAY` does not match the cases in `DatabricksTypeUtil.getColumnInfoType`, so the column type silently became `USER_DEFINED_TYPE`/`Types.OTHER`. Also, `DatabricksResultSetMetaData` crashed on `ColumnInfoTypeName.valueOf("TIMESTAMP_NTZ")` because that enum has no TIMESTAMP_NTZ constant. + **Rule:** Before mapping Databricks type text with `getColumnInfoType`/`ColumnInfoTypeName.valueOf`, strip the type parameters (`split("[<(]")`), take DECIMAL precision and scale from the declared type, and map TIMESTAMP_NTZ to TIMESTAMP. Add a metadata test for at least one parameterized element type and one non-STRING element type. +- **Context:** PR #1739. Engineer-bot follow-up runs failed again and again with `CalledProcessError` ("a non-bot commit appears to be at HEAD") while finalizing the bundle commit. The bot was re-triggered about five times per thread with the same result before one run marked the thread blocked, which flooded the PR with duplicate error and reviewer comments. When a later run did push a fix, Maven could not compile or test it because `spotless-maven-plugin` and `maven-jar-plugin` were missing from the offline cache. + **Rule:** When an engineer-bot follow-up hits the same bundle-commit `CalledProcessError` (non-bot commit at HEAD) more than once, stop re-running and mark the thread NEEDS HUMAN DECISION after the first repeat. When an edit is pushed without Maven being able to compile or run tests (offline cache missing plugins), say clearly that it is unverified. + +### 2026-10-30: learnings since 2026-10-29T17:25:30Z +- **Context:** PR #1752 made closed-connection checks consistent. Review found that `new DatabricksSQLException(reason, DatabricksDriverErrorCode)` does not emit the enum's numeric code. It derives the vendor code from the message text through `DatabricksVendorCode.getVendorCode(reason)`, which gives `0` for "Connection closed!". Inline copies of the same check in other overloads and classes (`DatabricksDatabaseMetaData`, `DatabricksPooledConnection`) kept reporting 0 while the shared helper reported 1010. + **Rule:** To surface a `DatabricksDriverErrorCode` as `getErrorCode()`, use the `(reason, sqlState, DatabricksDriverErrorCode)` constructor, not `(reason, DatabricksDriverErrorCode)`. Route every copy of a check through one shared helper. Tests should assert the numeric vendor code and must not pin it to 0. +- **Context:** PR #1751 bumped Guava from 33.0.0 to 33.7.2. Newer Guava pulls in `org.jspecify` and no longer brings `checker-qual` in transitively. The new namespace would have shipped unrelocated in the uber jar, and NOTICE was missing both libraries. + **Rule:** When a dependency bump adds or removes transitive packages, check `jar tf assembly-uber/target/*.jar` for raw non-`com.databricks` namespaces. Add a shade `` in `assembly-uber/pom.xml` and a NOTICE entry in the correct license section for each bundled library. +- **Context:** PR #1753 fixed offline (`mvn -o`) unit tests for the Engineer Bot follow-up workflow. Two things broke offline resolution. Surefire resolves `surefire-junit-platform` and `junit-platform-launcher` (version matched to the project's platform engine) lazily at test time, so `install -DskipTests` never caches them. Also, `_remote.repositories` files record the JFrog mirror ID (`jfrog-central`), which an empty offline settings.xml does not recognize. + **Rule:** Before switching Maven offline in CI, explicitly `dependency:get` the Surefire provider and the matching `junit-platform-launcher`. Rewrite the mirror repo ID in `~/.m2/**/_remote.repositories` to `central`. Keep warm-up and smoke-check steps best-effort so a broken PR branch doesn't block the bot from running. +- **Context:** On PR #1754, the SEA session-create replay fixtures still matched the old `/api/2.0/sql/sessions/` URL with `warehouse_id` in the body, but the driver now POSTs to `/api/2.0/sql/warehouses/{id}/sessions`. WireMock `bodyPatterns.equalTo` matches exactly, so any change to the URL or body breaks replay. + **Rule:** When changing a SEA request's URL or JSON body, grep `src/test/resources/sqlexecapi/**/mappings/*.json` for the old path. Update the `url` and exact `equalTo` body patterns, and delete mappings for behavior that was removed. +- **Context:** On PR #1742, the engineer-bot follow-up was re-triggered four times and failed the same way each time (`CalledProcessError` while finalizing the bundle commit, "non-bot commit at HEAD"). Only on the fifth run did it escalate as NEEDS HUMAN DECISION. The requested change was a few unit tests. + **Rule:** If a bot follow-up fails twice with the same internal or commit-setup error, stop re-triggering it. Mark the thread blocked or needs-human, and have a person clean up the branch or add the small fix by hand. + +### 2026-11-03: learnings since 2026-11-02T17:24:54Z +- **Context:** PR #1758 fixed intermittent `NoHttpResponseException` ("failed to respond") on SEA connections that sat idle for about 3 minutes. The Thrift client already closed pooled connections idle longer than `IdleHttpConnectionExpiry`. The SEA path, which uses the SDK `CommonsHttpClient` built in `ClientConfigurator`, never did, so it reused keep-alive sockets that the workspace front end was about to close (it drops idle connections at roughly 186s). The fix wraps the SDK `HttpClient` so it calls `closeIdleConnections(expiry, SECONDS)` before each request and skips the sweep when expiry <= 0. Review also asked for a test of the disabled (0) branch. The tests detect connection reuse by recording the client port seen by a local `HttpServer`. + **Rule:** Thrift and SEA build separate HTTP clients, so a connection-pool setting such as `IdleHttpConnectionExpiry` must be applied to both paths. When adding or fixing one, check `ClientConfigurator` (SEA/SDK) as well as the Thrift client builder. Test both the enabled branch and the disabled/0 branch; comparing client ports on a local server is a reliable way to assert whether a connection was reused. + +### 2026-11-05: learnings since 2026-11-04T17:24:59Z +- **Context:** PR #1623 added the routing boundary for native batches (`PreparedStatementBatchExecutor` → `NativeBatchExecutor`). The review bot asked for a generic `catch (RuntimeException)` that marks every row `EXECUTE_FAILED`, like the legacy path does. The author declined: once the backend has committed the batch, a later failure (for example, while reading results) would then be reported as a failed batch, and the caller might retry and write the rows twice. The native implementation (#1625) raises a separate `NativeBatchResultException` for post-execution failures, clears the batch, and checks that the update-count length matches the batch size. + **Rule:** In batch or other non-idempotent execution paths, do not wrap every exception as an all-`EXECUTE_FAILED` `BatchUpdateException`. Handle failures before submission and failures after the backend may have committed separately, so a committed batch is never reported as failed and retried. +- **Context:** PR #1623 sends parametrized INSERTs to native (server-side) batch execution. A reviewer pointed out that DBR rejects native batches whose INSERT has a multi-row `VALUES (...), (...)` clause. Routing that SQL shape through `addBatch()` to the native path would break statements that work today on the legacy path. There is no telemetry on how often customers use this shape, so the fix (detect it and fall back, or convert to single-row) was deferred until before native batching is turned on by default. + **Rule:** Native batch eligibility checks (such as `isParametrizedInsert`) must exclude multi-row `VALUES` INSERTs, or rewrite them to single-row, because DBR rejects them in native batches. Keep such SQL on the legacy path. + +### 2026-11-06: learnings since 2026-11-05T17:24:25Z +- **Context:** PR #1762 removed Thrift handle secrets from logs, `toString()` output and exception messages by sending statement IDs through `StatementId.loggableStatementId(...)`. Review then found that the code still stringified whole generated Thrift structs (`TStatus`, `TSparkDirectResults`, `TGetOperationStatusResp`). Those dumps pull in server stack traces (`infoMessages`), `errorDetailsJson`, binary fields and even inline metadata result rows. Any field added to the IDL later would also leak into user-facing SQLExceptions without anyone noticing. Some of these strings were also built eagerly on every successful call. + **Rule:** Never log or put a raw StatementId or a generated Thrift struct's `toString()` into an exception message. Use `StatementId.loggableStatementId(id)`, and pick out specific status fields (errorMessage, sqlState, errorCode, displayMessage) with a lazy formatter that only runs on the failure path. +- **Context:** In PR #1762, the test helper `LogCapture.publish` stored `record.getMessage()`, which is the raw JUL pattern. If a logger path passes arguments through `LogRecord.getParameters()`, those arguments never reach the captured text. Negative checks like `assertFalse(output.contains("secret:"))` would then pass even while a secret was being logged. + **Rule:** In log-capture test handlers, store `new SimpleFormatter().formatMessage(record)` rather than `record.getMessage()`. That way, negative "secret not logged" assertions can't pass just because parameters were never formatted into the text. +- **Context:** PR #1762 rewrote a log call that used a `{%s}` placeholder. `JulLogger` happens to render it because it calls `String.format` after replacing `{}`, but under the SLF4J backend the argument is silently dropped. + **Rule:** In JdbcLogger calls, use only `{}` placeholders, never `%s` or `{%s}`. Only `{}` works under both the JUL and SLF4J backends. + +### 2026-11-07: learnings since 2026-11-06T17:24:38Z +- **Context:** PR #1769 fixed flaky `DatabricksConnectionTest` evictor lifecycle tests. They had counted live JVM threads named "Connection evictor" against a baseline, then called `DatabricksHttpClientFactory.reset()` in `finally`. Any other test creating or closing HTTP clients in the same JVM changed the global thread count, and `reset()` wiped clients that other tests were still using. The fix gets the specific `IdleConnectionEvictor` from the client under test (via reflection), asserts `isRunning()` before close and `awaitTermination` plus `!isRunning()` after close, and cleans up with `removeClient(context)` only. + **Rule:** In lifecycle/leak tests, assert on the state of the specific resource handle the test created (e.g. `evictor.isRunning()`, `awaitTermination`), not on JVM-wide counts like `Thread.getAllStackTraces()` filtered by name. In `finally`, clean up only that test's entry in the shared singleton factory (`removeClient(ctx)`), never a global `reset()`. + +### 2026-11-11: learnings since 2026-11-10T17:25:04Z +- **Context:** In PR #1772 and PR #1776, connection opens stalled for tens of seconds. The cause was that `ConcurrentHashMap.compute*` lambdas did slow blocking work: the final telemetry flush in `TelemetryClient.close()`, building `ClientConfigurator` (auth/network), the first feature-flag fetch, and the feature-flag context `shutdown()`. A `compute*` call locks the whole hash bin, so connections to the same host waited, and so did connections whose keys only collide in the bin. The fix was to detach or create only a cheap holder inside the lambda and do the I/O or close after the lock is released, with a per-key lock where needed. + **Rule:** Never do network I/O, auth, `close()`/`shutdown()` or any other blocking call inside a `ConcurrentHashMap.compute`/`computeIfAbsent`/`computeIfPresent` lambda. Put a cheap holder in the map, or detach the value under the lock, and run the slow work outside it. To test this, use keys with colliding hashes ("Aa" and "BB" share a bin). +- **Context:** In PR #1776, `computeIfAbsent(key, k -> expensiveCreate())` was replaced with a lazily-filled holder that is created inside `computeIfAbsent` and filled outside it. Review found two regressions from that change. First, a failed creation left an empty holder in the static singleton map. `computeIfAbsent` stores nothing on failure, but the holder stayed, so every failed connect attempt (each with a fresh UUID) leaked an entry. Second, a re-entrant same-thread lookup (telemetry fired while building the exception) could run the one-time init, hit a recursive-creation error, and mark the host's feature flags as initialized with only defaults. The fixes: on failure, mark the holder closed and call `remove(key, holder)` so waiting threads retry; detect same-thread re-entry and return defaults without setting `initialized`; add cross-thread latch tests for both retry triggers. + **Rule:** When you replace `computeIfAbsent` with a holder plus lazy init, keep what `computeIfAbsent` guaranteed. A failed creation must remove the holder (`map.remove(key, holder)`) and let waiting threads retry. A re-entrant or failed init attempt must never mark the init complete. Test the failure path, the concurrent-removal path and the same-thread re-entry path. +- **Context:** PR #1772 bounded `Connection.close()` with `future.get(timeout)` on the final telemetry flush. Review found three problems. The timeout branch cancelled the future but the `InterruptedException` branch did not, so an abandoned push could keep holding a thread in the shared executor through its retry sleeps. Once the interrupt flag was restored, the next sequential close (the unauthenticated client) dropped its batch immediately. And the deadline counted time spent queued on the shared pool, plus each client's close got its own budget, so the documented caps were wrong unless the docs said so. + **Rule:** When giving up on a `Future.get(timeout)`, call `cancel(true)` in both the timeout and the interrupt branches, and restore the interrupt flag. Document that the timeout includes executor queue time, that it applies separately to each sequential wait, and that a restored interrupt makes later `get()` calls in the same close fail at once. + +### 2026-11-12: learnings since 2026-11-11T17:24:25Z +- **Context:** PR #1778 added a JVM-wide `DiscoveryResponseCache` for host-metadata and OIDC discovery documents. The documents are fetched through each connection's own SDK HttpClient. Review found that keying entries by URL alone let a document fetched under relaxed TLS (AllowSelfSignedCerts, a custom truststore, revocation checks off) be served to strict-TLS connections. That opens a cache-poisoning path to `token_endpoint`. The first fix also missed the `javax.net.ssl.trustStore`/`trustStoreType`/`trustStoreProvider` system properties. JSSE's default trust manager reads these again on the default `PoolingHttpClientConnectionManager` path, not only when `UseSystemTrustStore=1`. The final fix keys entries by `ConfiguratorUtils.trustSettingsKey`, which sits next to `getBaseConnectionManager`. A test changes each trust input one at a time and checks that the key changes, which enforces the keep-in-sync rule. + **Rule:** Any cross-connection/static cache of responses fetched through per-connection HTTP clients must include every TLS trust input in its key: driver truststore/keystore settings, self-signed/revocation flags, and the javax.net.ssl.* system properties, which are read on the default path too. Cache only requests without credentials (no Authorization or Cookie header), with no redirect to another origin or path, that return 200 with a body that parses. Add a test that changes each key input on its own and asserts the key changes. +- **Context:** PR #1778 added a TTL cache for discovery responses. Review caught three things. Production expiry used `System.currentTimeMillis`, so NTP or VM-resume clock jumps could stretch TTLs. No test exercised expiry. The cache grew without bound. The fix moved to a monotonic `nanoTime`-based clock, made the clock injectable, and added a test asserting the entry is still served at TTL-1 and is refetched and replaced at TTL. The cache ended up as a Guava `CacheBuilder` with `expireAfterWrite`, `maximumSize`, and an injectable `Ticker`. Single-flight on a cache miss was rejected on purpose. Each connection has its own proxy, truststore and timeout, so one connection's failed fetch must not be shared with others, and holding a per-key lock during network I/O would reintroduce the stalls fixed in #1776. + **Rule:** For in-JVM TTL caches, use a monotonic clock (nanoTime or a Guava Ticker), never wall-clock time. Bound the size (maximumSize plus expireAfterWrite). Make the clock injectable and test both sides of the TTL boundary. Do not add single-flight or per-key locks across network I/O when callers have different per-connection HTTP configs: allow duplicate fetches on a miss instead. + +### 2026-11-13: learnings since 2026-11-12T17:24:38Z +- **Context:** PR #1781 added exact `java.time` reads for pre-1582 dates. A reviewer found that `mayBeCalendarShifted` used a cutoff of cutover + 2 days. That missed DATE/TIMESTAMP values from 1582-10-05 to 1582-10-14, because `java.sql.Date`/`Timestamp` (hybrid Julian/Gregorian calendar) pushes those gap days forward by up to 10 days (1582-10-14 becomes 1582-10-24). The fix widened the window to cutover + 12 days and tested every gap day. + **Rule:** Julian/Gregorian cutover: any check that looks at a legacy `java.sql` Date/Timestamp `getTime()` to detect calendar shifting must cover cutover + ~12 days (the 10-day gap push plus a zone margin), and tests must cover every day from 1582-10-05 to 1582-10-14, not just one date on each side. +- **Context:** PR #1781 fixed nested ARRAY/STRUCT/MAP parsing failing with `Invalid TIMESTAMP_NTZ array representation`. Arrow serializes a nested TIMESTAMP_NTZ as a JSON component array `[y,m,d,h,min,s,nanos]` and drops trailing zero components, so a value with zero seconds has only 5 elements. The parser required at least 6. + **Rule:** Nested Arrow TIMESTAMP_NTZ component arrays drop trailing zero seconds/nanos: accept 5 to 7 elements, default the missing ones to 0, and test a value at exactly :00 seconds. +- **Context:** PR #1797 added the `EnableStrictGetObjectConversion` connection property, but only the `DatabricksResultSet` constructors that take an `IDatabricksSession` read it. DatabaseMetaData and in-memory result sets (e.g. `Array.getResultSet()`) are built without a session, so they silently kept the default. The reviewer flagged that the docs implied the property applied to every ResultSet. + **Rule:** A new connection property that changes ResultSet behavior does not reach session-less result sets (DatabaseMetaData, `Array.getResultSet()`, in-memory). Either pass the flag through explicitly, or state that it covers query results only in the property description, the Javadoc and NEXT_CHANGELOG. +- **Context:** In PR #1797, the strict `getObject(int, Class)` branch checked `type == null` before `checkIfClosed()`. On a closed result set it threw `UNSUPPORTED_OPERATION` instead of `RESULT_SET_CLOSED`, which breaks callers and telemetry that classify errors by SQLState or error code. + **Rule:** In JDBC getters, call `checkIfClosed()` before any argument validation or new early-exit branch, so a closed object always reports `RESULT_SET_CLOSED`. Pin the order with a test that passes an invalid argument on a closed object. +- **Context:** Several review rounds on PR #1781 found the same conversion behaving differently depending on the path. Type-map key lookup was exact-case at the top level but case-insensitive for nested types. 'Z'-suffixed TIMESTAMP text was moved to the JVM zone in the nested path but not at the top level. Unsupported `java.time` mappings threw on one path and returned null with a SQLWarning on another. + **Rule:** When adding a conversion to `getObject(col, Class/Map)`, make the top-level ResultSet, `Array.getArray(Map)`, `Struct.getAttributes(Map)` and `Array.getResultSet()` paths share one helper for lookup, zone handling and error behavior. Add a test that asserts nested and top-level results are equal, including with a non-UTC JVM default zone. + +### 2026-11-14: learnings since 2026-11-13T17:24:36Z +- **Context:** PR #1784 changed CloudFetch to stream the live HTTP body into `ArrowStreamReader`. Arrow reads through `Channels.newChannel(InputStream)`, which is an interruptible channel. When `shutdownNow()` interrupts a download worker, the JDK closes the channel's source stream on the *interrupting* thread (the one calling `ResultSet.close()`). On a live `ContentLengthInputStream`, that close drains the rest of the body over the network. Before the change the source was an in-memory array, so closing it was instant. The fix wraps the stream in `CloseShieldInputStream`, and only the download thread closes the body. + **Rule:** When Arrow (or any `Channels.newChannel` reader) wraps a live network/HTTP stream, shield it with `CloseShieldInputStream` so a thread interrupt can't close/drain the body on the interrupting thread; close it only from the reading thread, and use `response.close()`/abort (not drain) on failure paths. +- **Context:** In PR #1784, `AbstractArrowResultChunk.getRecordBatchList` caught `ClosedByInterruptException` and returned an empty `ArrowData` instead of rethrowing. That let an interrupted download be marked `DOWNLOAD_SUCCEEDED`/`PROCESSING_SUCCEEDED` with zero batches, so rows were truncated with no error. Partial batches were also purged only on `IOException`, so a `RuntimeException` (Arrow `OutOfMemoryException`, parse errors) leaked allocator memory, and `rootAllocator.close()` later threw. + **Rule:** After any Arrow read, check `Thread.currentThread().isInterrupted()` before marking a chunk successful. Decide cancellation from the interrupt flag, not the exception type (`SocketTimeoutException` is also an `InterruptedIOException` and must stay retryable). Purge partially-parsed Arrow buffers on `IOException | RuntimeException`, and assert `getAllocatedMemory() == 0` in the failure tests. +- **Context:** PRs #1790 and #1808 were perf-only changes: shared LZ4 decompressor/XXHash32 instances, and per-column converters cached in result-set metadata. A reviewer noted that the first checksum test also passed on `main`, so it could not catch a revert. The fix: warm up the shared instances, then open `mockStatic(Factory.class, CALLS_REAL_METHODS)` and `verify(..., never())` (or an exact `times(n)` at construction) on the per-call lookup. When the library itself legitimately calls the factory (`readHeader()` uses `XXHashFactory` for content-checksum frames), the test builds an input that avoids that path. PR #1783 tightened a one-sided bound (`delay < 300`) to `delay >= 290 && delay < 300` for the same reason: a broken computation that returned 0 would still have passed. + **Rule:** For perf/caching changes, add a regression test that FAILS on revert. Use a `CALLS_REAL_METHODS` static mock after warm-up to assert the hot-path lookup happens `never()`, and choose inputs that avoid legitimate library calls to it. Bound computed values (delays, TTLs) on both sides, and confirm the test fails against `main`. +- **Context:** PR #1783 added a JVM-wide static snapshot cache so new connections reuse recently fetched feature flags. Review found three problems: (1) the new lookup ran outside the fetch path's catch-all, so an NPE (e.g. `TreeMap` on a null header key) could fail `getConnection()` for best-effort data; (2) a reused snapshot could outlive the refresh interval a freshly fetched context would use (server TTL 3600 s vs. the forced 900 s), making flag flips stale for up to an hour; (3) static caches leak state between test classes unless they are cleared in setUp/tearDown. + **Rule:** When adding a cache in front of best-effort work (feature flags, telemetry, discovery), wrap the cache lookup in `catch (RuntimeException)` and fall back to the uncached path. Cap reuse at the same freshness bound the uncached path guarantees. Give any static/JVM-wide cache a `@VisibleForTesting` clear, and call it before and after every test that populates it. + +### 2026-11-16: learnings since 2026-11-15T17:24:32Z +- **Context:** PR #1633 added a WireMock request-body check that was meant to prove SEA sends ordered `parameter_sets`. The review found it used `equalToJson(json, true, true)` (ignoreArrayOrder=true, ignoreExtraElements=true) and left out the `value` fields, so a reordered or mislabeled batch would still have matched. + **Rule:** When a WireMock `equalToJson` assertion is meant to verify ordering or per-element contents, pass `ignoreArrayOrder=false` and include the distinguishing fields (e.g. `ordinal`, `value`). Expected elements that look structurally identical, or update counts that are all equal like `{1,1}`, cannot catch reordering. +- **Context:** PR #1625 (native Thrift batching). Reading update counts from a native batch result can fail after the backend has already committed the rows. The first version mapped that failure to `BatchUpdateException` with every count set to `EXECUTE_FAILED`. The fix added `NativeBatchResultException` (RESULT_SET_ERROR), which warns that rows may already be committed, and the code clears the stored batch before rethrowing. + **Rule:** Keep execution failures separate from post-success result-parsing failures. Never report `EXECUTE_FAILED` for work the server already committed, and clear any retryable client state (e.g. the stored JDBC batch) before propagating, so a caller retry cannot insert duplicate rows. +- **Context:** PR #1632 (native SEA parameter batching). The review found that `DatabricksSdkClient.executeStatement`'s PENDING/RUNNING poll loop calls `serializeStatementRequest(typedStatementId, request)`. For non-realtime sessions (`sessionId()==null`) that re-serializes the entire `ExecuteStatementRequest` as the body of every GET poll. With batching, that body now includes all `parameter_sets`. + **Rule:** In the SEA path, do not pass a large `ExecuteStatementRequest` into the shared poll loop without checking what each poll sends. Poll bodies built from the execute request grow with payload size. Prefer a lightweight `GetStatementRequest` (statement id only) for polling. +- **Context:** PR #1625 added `NativeBatchResultException`, which wraps a cause with a driver error code. The tests built the cause with the same SQLState (`RESULT_SET_ERROR`) the wrapper was supposed to set, so the assertion would also have passed if the wrapper just passed the cause's state through. Nothing checked the numeric vendor code either. + **Rule:** When testing an exception wrapper that assigns its own error code or SQLState, give the cause a different SQLState or code, and assert both the wrapper's error name and its numeric vendor code. Otherwise a pass-through bug cannot be detected. +- **Context:** PR #1625 added a new `IDatabricksClient.executeStatementBatch` method annotated `@DatabricksMetricsTimed`. Its latency was not recorded until the reviewer pointed out that it also has to be added to `TelemetryHelper.mapMethodToOperationType` (mapped to `OperationType.EXECUTE_STATEMENT`). + **Rule:** When adding a new `@DatabricksMetricsTimed` client method, register it in `TelemetryHelper.mapMethodToOperationType` and add a test for that mapping. Without the mapping, `recordOperationLatency` silently skips the method. + +### 2026-11-17: learnings since 2026-11-16T17:26:45Z +- **Context:** PR #1813 changed `DatabricksStatement.shouldReturnResultSet`/`isDescribableQuery`. They had treated any statement starting with `WITH` as a query, so `WITH ... INSERT/UPDATE/DELETE/MERGE` broke `executeUpdate()`. The fix finds the outer statement keyword by tracking paren depth outside CTE bodies, skipping `AS` and `MAX RECURSION LEVEL`. The same PR made `SqlCommentParser` follow Spark's `SqlBaseLexer`: backslash escapes inside quotes, `r'...'`/`r"..."` raw strings with no escapes, and `\`+newline continuing a `--` comment. + **Rule:** Never classify SQL as query vs. update from its leading keyword alone, since `WITH` can prefix DML. Any SQL scanning must follow Spark lexer quoting rules (backslash escapes, raw `r'...'` strings, backslash-continued line comments), not ANSI `''` escapes only. +- **Context:** PR #1811 rewrote the per-row hot path in `ArrowResultChunkIterator.nextRow()`/`hasNextRow()` for CPU. The test keeps a copy of the old algorithm (`PreviousIterator`) and compares the two step by step. It covers edge layouts (empty batches, chunk `numRows` below, equal to, and above the batch row total) plus 300 seeded random layouts, with and without calling `hasNextRow()` first. Existing quirks, such as an empty middle chunk making `next()` return false, are asserted as-is and documented instead of being fixed silently. + **Rule:** When optimizing a hot-path iterator or state machine, keep the old implementation in the test as an oracle. Differential-test it against the new one over edge and seeded-random inputs, including different call orders. Pin known existing bugs with explicit, commented assertions so the behavior doesn't change silently. diff --git a/.claude/knowledge/lessons.md b/.claude/knowledge/lessons.md new file mode 100644 index 000000000..ea7e047f5 --- /dev/null +++ b/.claude/knowledge/lessons.md @@ -0,0 +1,88 @@ +# Lessons + + + +## JDBC contracts and configuration +- L-0001 (1×, log:2026-08-14): Types.FLOAT maps to DOUBLE and 8-byte wire values; only Types.REAL is single precision. The declared SQL target type drives wire typing. +- L-0002 (1×, #1652): Merge Guava immutable maps in ascending configuration precedence and use buildKeepingLast so higher-priority values replace duplicates. +- L-0003 (1×, #1621): Native batch parameter ordinals are zero-based; legacy SEA/Thrift uses one-based JDBC indexes. Capture outbound requests on both paths. +- L-0004 (2×, #1641 #1678): Gate feature-specific request fields and behavior on the feature taking effect in the server response, preserving behavior for connections that did not opt in. +- L-0005 (1×, #1641): Apply behavioral and error-policy changes to every parallel metadata operation, or document each intentional divergence. +- L-0006 (1×, #1641): Validate server-controlled metadata column count and names before positional access, or resolve by name, to detect schema drift. +- L-0007 (1×, #1672): Explicit connection flags win; otherwise a default-enabled client flag defers to the server rollout flag, while a default-disabled feature stays off. +- L-0008 (2×, #1672 #1747): Document connection-property behavior and opt-out without exposing internal rollout flags. Removing a property also requires updating raw-key tests and documenting ignored legacy settings. +- L-0009 (1×, #1680): Preserve JDBC-required metadata ordering, including getTypeInfo ordering by DATA_TYPE, and test ordering separately from column contents. +- L-0010 (1×, #1683): Use Locale.ROOT when normalizing identifiers, hostnames and cache keys. +- L-0011 (1×, #1656): Add NEXT_CHANGELOG.md for user-visible changes, including dependency bumps; CI/tooling and documentation may use NO_CHANGELOG=true. +- L-0012 (3×, #1707): Share OAuth configuration parsing across U2M and M2M flows, treat blank overrides deliberately, and verify SDK forwarding in actual outbound requests. +- L-0013 (1×, #1713): Keep related accessors consistent on missing/null/invalid keys; guard null before Guava getIfPresent when the API promises an empty result. +- L-0014 (1×, #1728): Case-insensitive JDBC label lookup is first-wins. Use putIfAbsent after normalization so duplicate labels work consistently across SEA and Thrift. +- L-0015 (1×, #1746): Carry SEA routing state through SessionInfo, SessionId serialization/equality and StatementId; extend serialized IDs with optional trailing parts and test old/new round trips. +- L-0016 (1×, #1739): Compute JDBC long-position/int-count windows in long, clamp before casting, and test Integer.MAX_VALUE counts and positions beyond int range. +- L-0017 (1×, #1739): Strip type parameters before enum lookup, derive DECIMAL precision/scale from its declaration, and map TIMESTAMP_NTZ to JDBC TIMESTAMP. +- L-0018 (1×, #1797): ResultSet configuration must reach session-less metadata, Array and in-memory results, or its documented scope must explicitly cover query results only. +- L-0019 (1×, #1797): Call checkIfClosed before argument validation or early return in JDBC getters; closed objects must consistently report RESULT_SET_CLOSED. + +## Results and complex types +- L-0020 (2×, #1659 #1700): Handle null ARRAY/MAP/STRUCT elements before type-specific dispatch, for every element type. +- L-0021 (1×, #1700): Split complex-type metadata with depth- and quote-aware parsing; consumers must use structured parts instead of splitting flattened text again. +- L-0022 (1×, #1700): Defensively copy mutable arrays and collections on both input and output. +- L-0023 (1×, #1700): Share formatting across top-level, nested and feature-flag paths, including special cases such as omitted zero components. +- L-0024 (1×, #1700): When transforming one nested value type, preserve every other field's serialization and JSON-escape string leaves; domain toString is not container serialization. +- L-0025 (1×, #1731): SEA manifests may omit total_row_count. Continue via next_chunk_index and provider exhaustion; use an explicit unknown-count sentinel before row-count arithmetic. +- L-0026 (1×, #1731): Enumerate CloudFetch/SEA/Arrow flag combinations when changing provider routing; an unrelated false flag can accidentally select a downstream branch. +- L-0027 (1×, log:2026-10-22): Reproduce response-format bugs on the cluster/protocol that emits that format; a warehouse test cannot establish behavior of an APC-only response. +- L-0028 (2×, #1734): Select SEA result providers from actual response shape. URL_BASED_SET may contain inline Arrow with null links, including inline-plus-continuation responses. +- L-0029 (1×, log:2026-10-22): Parameterize complex-type E2E tests across SEA and Thrift with EnableComplexDatatypeSupport=1. +- L-0030 (1×, #1734): Advance the chunk provider successfully before updating ResultSet cursor state; failed advancement returns false without reading a nonexistent chunk. +- L-0031 (1×, log:2026-10-27): Use explicit catalog/schema in live metadata tests to avoid warehouse-wide scans; cover null-catalog behavior in unit tests. +- L-0032 (2×, #1623 #1625): Separate pre-submission, execution and post-commit parsing failures. Never report committed batch work as EXECUTE_FAILED, and clear retryable stored batch state before propagating errors. +- L-0033 (1×, #1623): Exclude multi-row VALUES INSERTs from native batching or rewrite to the backend's single-row form; unsupported shapes stay on the legacy path. +- L-0034 (1×, #1781): Calendar-cutover tests must cover every date from 1582-10-05 through 1582-10-14 and account for the gap plus a timezone margin. +- L-0035 (1×, #1781): Nested Arrow TIMESTAMP_NTZ arrays may have 5–7 components; default omitted seconds/nanos to zero and test exact-minute values. +- L-0036 (1×, #1781): Share conversion lookup, zone handling and errors across ResultSet and nested Array/Struct accessors; compare nested/top-level behavior under a non-UTC JVM zone. +- L-0037 (1×, #1784): Shield Arrow-wrapped live HTTP streams from interrupt-driven channel close; let the reading thread close them and abort failed responses instead of draining. +- L-0038 (1×, #1784): After Arrow reads, check the interrupt flag before success. SocketTimeoutException remains retryable; purge partial buffers on IO/runtime failure and assert zero allocated memory. +- L-0039 (1×, #1632): SEA polling should send a lightweight statement-id request; reusing a large execute request can resend its payload on every poll. +- L-0040 (1×, #1813): SQL classification must handle WITH-prefixed DML and Spark lexical rules, including backslash escapes, raw strings and continued line comments. + +## Errors and telemetry +- L-0041 (4×, #1629 #1739 #1752): Use registered DatabricksDriverErrorCode values and the SQLException constructor that preserves numeric vendor code. Assert SQLSTATE/name/code with a distinguishable cause and record telemetry classification. +- L-0042 (1×, #1670): Compare JUL levels by intValue; equivalent levels need not share object identity. +- L-0043 (2×, #1683): Preserve the original error across recovery failures, including unchecked failures; inspect getSuppressed when suppression holds the original exception. +- L-0044 (1×, #1709): UserAgent.sanitize performs replacement without IllegalArgumentException; withOtherInfo key validation is the throwing operation. +- L-0045 (1×, #1762): Log statement IDs through loggableStatementId and select status fields lazily; raw generated Thrift toString output can expose secrets. +- L-0046 (1×, #1762): Capture fully formatted JUL messages with SimpleFormatter.formatMessage so negative secret-logging tests cannot pass on unexpanded parameters. +- L-0047 (1×, #1762): JdbcLogger uses {} placeholders under both JUL and SLF4J; avoid printf placeholders. +- L-0048 (1×, #1625): Register each new DatabricksMetricsTimed client method in TelemetryHelper.mapMethodToOperationType and test the mapping. + +## Concurrency and caching +- L-0049 (1×, #1670): Mark singleton initialization complete only after useful configuration is installed; disabled, failed or reentrant attempts must remain retryable. +- L-0050 (2×, #1683 #1778 #1776): Bound TTL caches by size and freshness, use an injectable monotonic clock, and test expiry boundaries. Avoid network single-flight across differing connection HTTP configurations. +- L-0051 (2×, #1683 #1783): Best-effort caches must fall back safely on runtime failures and preserve uncached freshness guarantees. Clear shared caches around tests; warn when recovery overrides explicit user configuration. +- L-0052 (1×, #1678): Choose one synchronization protocol per field; avoid redundant atomics inside monitor-protected mutation paths. +- L-0053 (2×, #1709 #1726): SDK UserAgent registration is process-global and append-only. Register each distinct entry once, bound growth, and restore global state in tests. +- L-0054 (1×, #1726): SEA and Thrift obtain User-Agent through different paths; verify both transports after changing shared registration. +- L-0055 (1×, #1758): Apply connection-pool settings to both SEA and Thrift HTTP clients; test enabled and disabled/zero behavior through observed connection reuse. +- L-0056 (1×, #1769): Leak tests must inspect their own resource handles and clean only their singleton entries, avoiding global thread counts and global resets. +- L-0057 (1×, #1772 #1776): Keep network, auth, close and shutdown work outside ConcurrentHashMap compute lambdas; test unrelated colliding keys to expose lock contention. +- L-0058 (1×, #1776): Lazy-init holders must preserve computeIfAbsent failure/retry semantics. Remove failed holders conditionally and test concurrent removal and same-thread reentry. +- L-0059 (1×, #1772): Cancel timed-out or interrupted futures, restore the interrupt flag, and document queue time and separate sequential wait budgets. +- L-0060 (1×, #1778): Shared HTTP-response cache keys must include every TLS trust input. Cache only valid 200 responses without credentials or cross-origin/path redirects; vary each key input in tests. + +## Verification and dependencies +- L-0061 (1×, #1582): Required workflows must handle merge_group. Skip jobs requiring pull_request fields in queue events while retaining their PR-time gate. +- L-0062 (2×, #1663 #1751): Upgrade coupled HTTP libraries as a published-compatible set and exercise HTTP paths. Inspect shaded JAR namespaces, adding relocations and NOTICE entries for newly bundled dependencies. +- L-0063 (2×, #1641 #1700): Exercise the real caller path and compare against an independent reference; wrappers delegating to the same helper do not prove equivalence. +- L-0064 (2×, #1703 #1754): Update all SEA WireMock paths and exact request bodies when wire behavior changes, removing obsolete mappings. +- L-0065 (1×, #1723): Avoid HTTP gzip over already-compressed LZ4 Arrow by default; use identity encoding and make recompression an explicit bandwidth tradeoff. +- L-0066 (1×, #1700): Decode real producer wire fixtures; round trips through the same writer can hide production serialization quirks. +- L-0067 (1×, #1753): Before offline Maven tests, warm Surefire providers and matching junit-platform-launcher, normalize repository IDs, and keep bot warm-up checks best-effort. +- L-0068 (2×, #1790 #1808 #1811): Performance regressions must fail on revert. Use warmed hot-path call assertions and differential tests against the old iterator/state machine, including random inputs and call orders. +- L-0069 (1×, #1633): Ordering tests need ignoreArrayOrder=false and distinguishable per-element values; identical elements or update counts cannot detect reordering. + +## Bot workflow +- L-0070 (1×, log:2026-08-14): Keep diagnosis/plan phases read-only; defer file creation and edits to the author_tests/fix phases. +- L-0071 (3×, #1582 #1739 #1742): State agreed environment restrictions once. Stop repeated identical bot setup/commit failures, request human resolution, and label uncompiled or untested edits as unverified. +- L-0072 (1×, log:2026-10-22): Author-run environments lack WebFetch/WebSearch; diagnose from repository context and available tools. +- L-0073 (1×, log:2026-10-22): Carry exact file locations, signatures and helper findings from diagnosis into implementation prompts to avoid repeated discovery. diff --git a/.github/actions/bot-prelude/action.yml b/.github/actions/bot-prelude/action.yml index b565fccec..2c9351657 100644 --- a/.github/actions/bot-prelude/action.yml +++ b/.github/actions/bot-prelude/action.yml @@ -30,7 +30,7 @@ inputs: # value to move every bot to a new engine commit; never @main. description: 'Engine commit SHA (full 40-char) to install.' required: false - default: 'ac3b08eab993ef14bf997a9bf4c776fa5acb16d8' + default: '60144783c5c3e4c242df9f1f5a0f25cd71cf5f4c' engine-repo: description: 'owner/name of the engine repo.' required: false diff --git a/.github/workflows/engineer-bot-learning.yml b/.github/workflows/engineer-bot-learning.yml index 82cab249c..dd349f7ea 100644 --- a/.github/workflows/engineer-bot-learning.yml +++ b/.github/workflows/engineer-bot-learning.yml @@ -1,28 +1,5 @@ -# Engineer Bot — learning (retrospective) extraction — DAILY CRON. -# -# Over an adaptive look-back window the engine gathers merged PRs (diff + review -# comments) AND recent engineer-bot author-run console logs ITSELF via the GitHub -# API — no in-workflow context gathering, no per-PR trigger — and if the model -# finds durable, reusable learnings, opens ONE ROLLING PR on a stable branch -# (`ai/learning-pr`), appending a dated section per day until a human merges it. -# Human-gated by design: it NEVER commits the canonical log directly. -# -# That stable branch is DERIVED from .bot/config.yaml `retrospective.branch_prefix` -# (`ai/learning-pr-`, trailing separator): the engine's `stable_branch()` strips -# the trailing `-/ ` so the prefix collapses to the single date-less branch -# `ai/learning-pr`. It is intentionally date-LESS — at most one open learning PR -# ever exists; there is no per-run suffix and no branch-per-run churn. The config -# keeps the `-` suffix to match the reference (sql-python); the two are NOT in -# conflict — prefix in config, derived branch here. -# -# Own job (NOT `uses: databricks/databricks-bot-engine/...`): an external repo -# can't resolve the internal engine's reusable workflows ("not found"). It shares -# the SAME prelude the other bots use — ./.github/actions/bot-prelude (tokens + -# Node + pinned engine install) — so the engine pin stays single-sourced in -# bot-prelude's `engine-ref` default (no second SHA to drift). -# -# Opt-in is purely via the `retrospective:` block in .bot/config.yaml + this -# workflow; absent that block the engine phase is a clean no-op. +# Daily journal extraction and lesson distillation in a rolling PR. +# The legacy log is frozen; learning PRs require human review. name: Engineer Bot — Learning on: diff --git a/.github/workflows/reviewer-bot.yml b/.github/workflows/reviewer-bot.yml index cb5e5b5ee..34390a687 100644 --- a/.github/workflows/reviewer-bot.yml +++ b/.github/workflows/reviewer-bot.yml @@ -60,12 +60,11 @@ jobs: labels: linux-ubuntu-latest timeout-minutes: 30 steps: - # Checkout FIRST. Default ref = the PR merge ref on pull_request, the - # default branch on dispatch. persist-credentials:false — the reviewer - # reads this tree via read_paths/grep, so no token may sit in .git/config. - - name: Checkout + # Run trusted workflow code; inspect the resolved PR head separately. + - name: Checkout trusted workflow source uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} fetch-depth: 0 persist-credentials: false @@ -110,8 +109,19 @@ jobs: fi [[ "$RAW_PR" =~ ^[0-9]+$ ]] || { echo "::error::Invalid pr_number '$RAW_PR'"; exit 1; } case "$DRY_RUN" in true|false) ;; *) DRY_RUN="true" ;; esac # fail safe - HEAD_SHA=$(gh pr view "$RAW_PR" --repo "$REPO" --json headRefOid -q .headRefOid) - { echo "pr_number=$RAW_PR"; echo "head_sha=$HEAD_SHA"; echo "dry_run=$DRY_RUN"; } >> "$GITHUB_OUTPUT" + PR_JSON=$(gh api "repos/$REPO/pulls/$RAW_PR") + jq -e --arg repo "$REPO" '.state == "open" and .head.repo.full_name == $repo and .head.repo.fork == false' <<< "$PR_JSON" >/dev/null \ + || { echo "::error::Reviewer requires an open PR from this repository"; exit 1; } + HEAD_SHA=$(jq -r '.head.sha' <<< "$PR_JSON") + BASE_SHA=$(jq -r '.base.sha' <<< "$PR_JSON") + { echo "pr_number=$RAW_PR"; echo "head_sha=$HEAD_SHA"; echo "base_sha=$BASE_SHA"; echo "dry_run=$DRY_RUN"; } >> "$GITHUB_OUTPUT" + + - name: Checkout reviewed PR (read-only) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ steps.inputs.outputs.head_sha }} + path: review-content + persist-credentials: false - name: Run reviewer env: @@ -119,7 +129,9 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ steps.inputs.outputs.pr_number }} HEAD_SHA: ${{ steps.inputs.outputs.head_sha }} + PR_BASE_SHA: ${{ steps.inputs.outputs.base_sha }} DRY_RUN: ${{ steps.inputs.outputs.dry_run }} + REVIEW_CONTENT_ROOT: ${{ github.workspace }}/review-content REVIEWER_BOT_MODEL_OVERRIDE: databricks-claude-opus-5-5 MODEL_ENDPOINT: https://${{ secrets.DATABRICKS_HOST }}/serving-endpoints/databricks-claude-opus-5-5/invocations DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}