From f6dcf33a169250c8bb2631c6121749dbdf52bf64 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Mon, 26 Oct 2026 22:13:54 -0700 Subject: [PATCH 1/6] feat(lance): add Lance-backed graph storage extension Implements the icebug-disk storage design (docs: node datasets, CSR indices+indptr datasets, flat rel datasets, immutable tables) on Lance datasets instead of Parquet files: - LANCE_SCAN table function (also enables LOAD FROM / COPY FROM with file_format='lance') reading through DuckDB's lance core extension, the same engine built by lance-duckdb. - LanceNodeTable / LanceRelTable (CSR + FLAT) on ColumnarNodeTableBase / ColumnarRelTableBase, registered via ExternalTableHookRegistry for StorageFormat::LANCE; range-limited CSR scans, degree summaries, PK lookup and recursive traversal all work through the shared columnar scan machinery. - e2e coverage (test/test_files/lance.test, fixtures mirror dataset/ice-disk-test row-for-row), path unit tests, and a fixture generator script (scripts/gen_test_data.sh). Requires the core external-table-hooks API (stacked ladybug PR); validated locally against it (9 e2e cases + 12 unit tests green). --- .github/workflows/ci.yml | 2 +- CMakeLists.txt | 1 + extension_config.cmake | 2 +- lance/CMakeLists.txt | 39 ++ lance/README.md | 120 ++++ lance/scripts/gen_test_data.sh | 77 ++ lance/src/connector/CMakeLists.txt | 9 + lance/src/connector/lance_connector.cpp | 83 +++ lance/src/function/CMakeLists.txt | 10 + lance/src/function/lance_scan.cpp | 82 +++ lance/src/include/connector/lance_connector.h | 35 + lance/src/include/function/lance_scan.h | 50 ++ lance/src/include/main/lance_extension.h | 17 + lance/src/include/storage/lance_constants.h | 34 + lance/src/include/storage/lance_node_table.h | 110 +++ lance/src/include/storage/lance_reader.h | 83 +++ lance/src/include/storage/lance_rel_table.h | 96 +++ lance/src/include/storage/lance_utils.h | 67 ++ lance/src/installer/CMakeLists.txt | 11 + lance/src/installer/lance_install_func.cpp | 16 + lance/src/main/CMakeLists.txt | 9 + lance/src/main/lance_extension.cpp | 77 ++ lance/src/storage/CMakeLists.txt | 9 + lance/src/storage/lance_node_table.cpp | 332 +++++++++ lance/src/storage/lance_reader.cpp | 176 +++++ lance/src/storage/lance_rel_table.cpp | 658 ++++++++++++++++++ lance/test/CMakeLists.txt | 5 + ...0-d9417151-fab9-469f-8983-c86ca9730200.txn | Bin 0 -> 186 bytes .../_versions/18446744073709551614.manifest | Bin 0 -> 450 bytes .../_versions/latest_version_hint.json | 1 + ...10101111110d66a94658a1681e2dd8cb3351.lance | Bin 0 -> 647 bytes ...0-f43af6ed-319f-4763-be43-5356d559fb7c.txn | Bin 0 -> 150 bytes .../_versions/18446744073709551614.manifest | Bin 0 -> 379 bytes .../_versions/latest_version_hint.json | 1 + ...0011110111230435463bad251acd84149894.lance | Bin 0 -> 357 bytes ...0-4c1da2c1-d8ee-4b6f-a1dc-6169d17169bd.txn | Bin 0 -> 147 bytes .../_versions/18446744073709551614.manifest | Bin 0 -> 373 bytes .../_versions/latest_version_hint.json | 1 + ...001001110096faa84adeac97555244998153.lance | Bin 0 -> 344 bytes ...0-9d2257f5-6204-40ca-af23-c8db786c4549.txn | Bin 0 -> 147 bytes .../_versions/18446744073709551614.manifest | Bin 0 -> 373 bytes .../_versions/latest_version_hint.json | 1 + ...1101011001da68aa4c8790c77993f080e465.lance | Bin 0 -> 344 bytes ...0-c9bd59dd-a0f6-492b-9928-b493e3851b49.txn | Bin 0 -> 222 bytes .../_versions/18446744073709551614.manifest | Bin 0 -> 522 bytes .../_versions/latest_version_hint.json | 1 + ...1100010011a80be542a0a09797b48042e19a.lance | Bin 0 -> 932 bytes ...0-f653a29d-2a71-46c3-8673-e2aae3d3b7e8.txn | Bin 0 -> 215 bytes .../_versions/18446744073709551614.manifest | Bin 0 -> 508 bytes .../_versions/latest_version_hint.json | 1 + ...010010000114cc1a4c65bbf919ce5f1ef63b.lance | Bin 0 -> 925 bytes ...0-3cdb6ab2-973b-49a6-9deb-16b5d7c90f6a.txn | Bin 0 -> 216 bytes .../_versions/18446744073709551614.manifest | Bin 0 -> 510 bytes .../_versions/latest_version_hint.json | 1 + ...1111111111503add472da3e238fbab38a584.lance | Bin 0 -> 942 bytes lance/test/lance_utils_test.cpp | 61 ++ lance/test/test_files/lance.test | 225 ++++++ 57 files changed, 2501 insertions(+), 2 deletions(-) create mode 100644 lance/CMakeLists.txt create mode 100644 lance/README.md create mode 100755 lance/scripts/gen_test_data.sh create mode 100644 lance/src/connector/CMakeLists.txt create mode 100644 lance/src/connector/lance_connector.cpp create mode 100644 lance/src/function/CMakeLists.txt create mode 100644 lance/src/function/lance_scan.cpp create mode 100644 lance/src/include/connector/lance_connector.h create mode 100644 lance/src/include/function/lance_scan.h create mode 100644 lance/src/include/main/lance_extension.h create mode 100644 lance/src/include/storage/lance_constants.h create mode 100644 lance/src/include/storage/lance_node_table.h create mode 100644 lance/src/include/storage/lance_reader.h create mode 100644 lance/src/include/storage/lance_rel_table.h create mode 100644 lance/src/include/storage/lance_utils.h create mode 100644 lance/src/installer/CMakeLists.txt create mode 100644 lance/src/installer/lance_install_func.cpp create mode 100644 lance/src/main/CMakeLists.txt create mode 100644 lance/src/main/lance_extension.cpp create mode 100644 lance/src/storage/CMakeLists.txt create mode 100644 lance/src/storage/lance_node_table.cpp create mode 100644 lance/src/storage/lance_reader.cpp create mode 100644 lance/src/storage/lance_rel_table.cpp create mode 100644 lance/test/CMakeLists.txt create mode 100644 lance/test/lance_data/indices_follows.lance/_transactions/0-d9417151-fab9-469f-8983-c86ca9730200.txn create mode 100644 lance/test/lance_data/indices_follows.lance/_versions/18446744073709551614.manifest create mode 100644 lance/test/lance_data/indices_follows.lance/_versions/latest_version_hint.json create mode 100644 lance/test/lance_data/indices_follows.lance/data/0000101010000110101111110d66a94658a1681e2dd8cb3351.lance create mode 100644 lance/test/lance_data/indices_livesin.lance/_transactions/0-f43af6ed-319f-4763-be43-5356d559fb7c.txn create mode 100644 lance/test/lance_data/indices_livesin.lance/_versions/18446744073709551614.manifest create mode 100644 lance/test/lance_data/indices_livesin.lance/_versions/latest_version_hint.json create mode 100644 lance/test/lance_data/indices_livesin.lance/data/111000001111010011110111230435463bad251acd84149894.lance create mode 100644 lance/test/lance_data/indptr_follows.lance/_transactions/0-4c1da2c1-d8ee-4b6f-a1dc-6169d17169bd.txn create mode 100644 lance/test/lance_data/indptr_follows.lance/_versions/18446744073709551614.manifest create mode 100644 lance/test/lance_data/indptr_follows.lance/_versions/latest_version_hint.json create mode 100644 lance/test/lance_data/indptr_follows.lance/data/01100111110111001001110096faa84adeac97555244998153.lance create mode 100644 lance/test/lance_data/indptr_livesin.lance/_transactions/0-9d2257f5-6204-40ca-af23-c8db786c4549.txn create mode 100644 lance/test/lance_data/indptr_livesin.lance/_versions/18446744073709551614.manifest create mode 100644 lance/test/lance_data/indptr_livesin.lance/_versions/latest_version_hint.json create mode 100644 lance/test/lance_data/indptr_livesin.lance/data/011010100111001101011001da68aa4c8790c77993f080e465.lance create mode 100644 lance/test/lance_data/nodes_city.lance/_transactions/0-c9bd59dd-a0f6-492b-9928-b493e3851b49.txn create mode 100644 lance/test/lance_data/nodes_city.lance/_versions/18446744073709551614.manifest create mode 100644 lance/test/lance_data/nodes_city.lance/_versions/latest_version_hint.json create mode 100644 lance/test/lance_data/nodes_city.lance/data/111101101100001100010011a80be542a0a09797b48042e19a.lance create mode 100644 lance/test/lance_data/nodes_user.lance/_transactions/0-f653a29d-2a71-46c3-8673-e2aae3d3b7e8.txn create mode 100644 lance/test/lance_data/nodes_user.lance/_versions/18446744073709551614.manifest create mode 100644 lance/test/lance_data/nodes_user.lance/_versions/latest_version_hint.json create mode 100644 lance/test/lance_data/nodes_user.lance/data/11010111000101010010000114cc1a4c65bbf919ce5f1ef63b.lance create mode 100644 lance/test/lance_data/rels_follows_flat.lance/_transactions/0-3cdb6ab2-973b-49a6-9deb-16b5d7c90f6a.txn create mode 100644 lance/test/lance_data/rels_follows_flat.lance/_versions/18446744073709551614.manifest create mode 100644 lance/test/lance_data/rels_follows_flat.lance/_versions/latest_version_hint.json create mode 100644 lance/test/lance_data/rels_follows_flat.lance/data/010010001001101111111111503add472da3e238fbab38a584.lance create mode 100644 lance/test/lance_utils_test.cpp create mode 100644 lance/test/test_files/lance.test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bc65ff3..a69a3a5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ concurrency: env: # pg_client requires libpq (PostgreSQL) at build time and a running # PostgreSQL instance at test time (provided by run_pgembed_fixture.py). - EXTENSION_LIST: adbc;azure;delta;duckdb;ducklake;fts;httpfs;iceberg;json;llm;neo4j;pg_client;postgres;sqlite;unity_catalog;vector;algo;gql + EXTENSION_LIST: adbc;azure;delta;duckdb;ducklake;fts;httpfs;iceberg;json;llm;neo4j;pg_client;postgres;sqlite;unity_catalog;vector;algo;gql;lance jobs: # ───────────────────────────────────────────────────────────────── diff --git a/CMakeLists.txt b/CMakeLists.txt index aebf4153..a01ad155 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,6 +87,7 @@ add_extension_if_enabled_and_skip_32bit("iceberg") add_extension_if_enabled_and_skip_32bit("azure") add_extension_if_enabled_and_skip_32bit("unity_catalog") add_extension_if_enabled_and_skip_32bit("ducklake") +add_extension_if_enabled_and_skip_32bit("lance") add_extension_if_enabled("json") add_extension_if_enabled("fts") add_extension_if_enabled("vector") diff --git a/extension_config.cmake b/extension_config.cmake index 63d2a8aa..4345e217 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -1,4 +1,4 @@ -set(EXTENSION_LIST adbc azure delta duckdb ducklake fts httpfs iceberg json llm pg_client postgres sqlite unity_catalog vector neo4j algo gql) +set(EXTENSION_LIST adbc azure delta duckdb ducklake fts httpfs iceberg json lance llm pg_client postgres sqlite unity_catalog vector neo4j algo gql) #set(EXTENSION_STATIC_LINK_LIST fts) foreach(extension IN LISTS EXTENSION_STATIC_LINK_LIST) diff --git a/lance/CMakeLists.txt b/lance/CMakeLists.txt new file mode 100644 index 00000000..104688ed --- /dev/null +++ b/lance/CMakeLists.txt @@ -0,0 +1,39 @@ +if (WIN32 OR BUILD_STATIC_EXTENSION) + set(DuckDB_USE_STATIC_LIBS ON) +else () + set(DuckDB_USE_STATIC_LIBS OFF) +endif () +find_package(DuckDB REQUIRED) + +include_directories( + ${PROJECT_SOURCE_DIR}/src/include + ${CMAKE_BINARY_DIR}/src/include + src/include + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/include + ${PROJECT_SOURCE_DIR}/extension/httpfs/src/include + ${DuckDB_INCLUDE_DIRS} +) + +add_subdirectory(src/connector) +add_subdirectory(src/function) +add_subdirectory(src/storage) +add_subdirectory(src/installer) +add_subdirectory(src/main) +add_subdirectory(test) + +build_extension_lib(${BUILD_STATIC_EXTENSION} "lance") + +target_link_libraries(lbug_${EXTENSION_LIB_NAME}_extension + PRIVATE + ${DuckDB_LIBRARIES}) + +if (NOT WIN32 AND NOT BUILD_STATIC_EXTENSION) + add_library(lance_loader + SHARED + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/loader/duckdb_loader.cpp) + set_extension_properties(lance_loader lance_loader lance) +endif () + +if (APPLE AND NOT BUILD_STATIC_EXTENSION) + set_apple_dynamic_lookup(lance_loader) +endif () diff --git a/lance/README.md b/lance/README.md new file mode 100644 index 00000000..fd94d571 --- /dev/null +++ b/lance/README.md @@ -0,0 +1,120 @@ +# Lance Extension + +Lance-backed read-only graph storage for Ladybug: node tables and CSR (or flat) +relationship tables whose data lives in [Lance](https://lancedb.github.io/lance/) +datasets instead of Ladybug's native storage engine. + +It is the Lance counterpart of the **icebug-disk** format (`docs/icebug-disk.md`): +same table layout and Cypher semantics, but every backing file is a Lance +dataset instead of a Parquet file. + +## Quick start + +```cypher +LOAD EXTENSION 'extension/lance/build/liblance.lbug_extension'; + +CREATE NODE TABLE user(id INT32 PRIMARY KEY, name STRING, age INT64) + WITH (storage = 'graph/', format = 'lance'); +CREATE NODE TABLE city(id INT32 PRIMARY KEY, name STRING, population INT64) + WITH (storage = 'graph/', format = 'lance'); +CREATE REL TABLE follows(FROM user TO user, since INT32) + WITH (storage = 'graph/', format = 'lance'); +CREATE REL TABLE livesin(FROM user TO city) + WITH (storage = 'graph/', format = 'lance'); + +MATCH (a:user {id: 100})-[:follows]->(b)-[:livesin]->(c:city) +RETURN b.name, c.name ORDER BY b.name; +``` + +Expected files (see `docs/lance.md` for the full layout): + +``` +graph/nodes_user.lance graph/nodes_city.lance +graph/indices_follows.lance graph/indptr_follows.lance +graph/indices_livesin.lance graph/indptr_livesin.lance +``` + +A relationship table whose `storage` points directly at a `.lance` +dataset uses the FLAT layout (`src, dst, ` rows) instead of CSR. + +Ad-hoc reads without creating tables: + +```cypher +CALL LANCE_SCAN('graph/nodes_user.lance') RETURN *; +LOAD FROM 'graph/nodes_user.lance' (file_format = 'lance') RETURN *; +COPY person FROM 'graph/nodes_user.lance' (file_format = 'lance'); +``` + +## How it works + +**Lance I/O goes through DuckDB.** The extension embeds an in-memory DuckDB +that loads the `lance` core extension — i.e. the exact engine built by +[lance-duckdb](https://github.com/lance-duckdb) (this extension tracks what +DuckDB `v2.0-cyanoptera` ships, the same line lance-duckdb `main` targets). +Reads are range queries (`SELECT FROM '.lance' LIMIT/OFFSET`) +which lance-duckdb pushes into the Lance scan, so morsel and CSR range scans +stay efficient. This mirrors how the `delta`/`iceberg` extensions reuse +DuckDB's Delta/Iceberg extensions, and it means no Rust toolchain or Lance +linkage is required to build or run this extension. + +**Storage layout mirrors icebug-disk** (`IceDiskNodeTable`/`IceDiskRelTable` +are the templates for `LanceNodeTable`/`LanceRelTable`): positional node +offsets, `indices` (edge targets + properties, CSR-sorted by source) plus a +single-column `indptr`, indptr cached in memory and range-limiting each +bound-node batch from O(edges) to O(matching edges) on forward scans — the +same optimization icebug-disk applies to Parquet row groups. Like icebug-disk +v1, tables are **immutable** (inserts/updates/deletes are rejected) and +edge properties match **by name** (column order is insignificant; missing +properties read as NULL). + +**Plugging into core.** Storage table types cannot live behind a plain table +function (graph traversal needs CSR extend, degree summaries, PK lookup), so +core provides a small extension hook surface (see +`src/include/storage/table/external_table_hooks.h`): + +- `StorageFormat::LANCE` + `ExternalTableHookRegistry` (factories + scan hooks, + keyed by format and by table); +- the scan operators (`ScanNodeTable`, `ScanRelTable`, `ScanMultiRelTable`, + `OnDiskGraph` for recursive traversal) create scan states and count morsels + through the hooks, falling back to the legacy paths otherwise; +- `LanceRelTableScanState` extends `IceDiskRelTableScanState` so one shared + state can scan lance, icebug-disk, Arrow, and native tables in multi-rel + scans (rank 3 in the coverage lattice). + +No core query path changed behavior: all pre-existing icebug-disk/Arrow/native +tests pass unmodified. + +## Relationship to lance-csr + +[lance-csr](https://github.com/hyperdimensionalcomputingai/lance-csr) is a +Cypher engine over Lance datasets with an **in-memory** CSR index built from +edge tables at query time. This extension takes the complementary, +storage-first approach: the CSR (`indices` + `indptr`) is **materialized as +Lance datasets on disk** (the icebug-disk layout translated to Lance), so +graphs are portable files that any Lance reader can open, while traversal +still resolves hops through CSR range lookups instead of joins. A native +(Rust, zero-DuckDB) reader reusing lance-duckdb's `rust/` crates is the +natural next step; the table layer is already decoupled behind `LanceReader`. + +## Regenerating test fixtures + +```bash +./extension/lance/scripts/gen_test_data.sh +``` + +regenerates `extension/lance/test/lance_data/` with the DuckDB CLI. Content +mirrors `dataset/ice-disk-test` row-for-row. + +## Limitations (v1) + +- Read-only tables; no `COPY ... TO ... (FORMAT lance)` write path yet (write + Lance datasets with DuckDB/pylance, or vote for the native writer). +- `STRUCT`/`MAP`/`UNION` property columns are rejected at `CREATE` with a + clear error; scalars, `LIST`/`ARRAY` (incl. fixed-size vectors), temporal, + `UUID`, `DECIMAL`, and `BLOB` are supported. +- Object-store datasets skip validation at `CREATE` (credentials may be + configured later) and fail at first scan, mirroring icebug-disk. +- Semi-mask morsel skipping is not implemented (masks still filter); GDS + vertex scans (`scanVertices`) over Lance tables are not supported. +- `LOAD` the extension before creating Lance tables; the "load the extension" + error otherwise points at the fix. diff --git a/lance/scripts/gen_test_data.sh b/lance/scripts/gen_test_data.sh new file mode 100755 index 00000000..4f6d1cd8 --- /dev/null +++ b/lance/scripts/gen_test_data.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Regenerates the Lance test datasets under extension/lance/test/lance_data/. +# Uses the DuckDB CLI with its lance core extension (the same engine the lance +# extension reads through), so the files are always readable by the extension. +# Content mirrors dataset/ice-disk-test (the icebug-disk fixtures) row-for-row, +# letting the e2e expectations mirror test/test_files/ice_disk/. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="${SCRIPT_DIR}/../test/lance_data" +rm -rf "${OUT_DIR}" +mkdir -p "${OUT_DIR}" + +duckdb -unsigned :memory: < + PARENT_SCOPE) diff --git a/lance/src/connector/lance_connector.cpp b/lance/src/connector/lance_connector.cpp new file mode 100644 index 00000000..503f3a9d --- /dev/null +++ b/lance/src/connector/lance_connector.cpp @@ -0,0 +1,83 @@ +#include "connector/lance_connector.h" + +#include "common/exception/runtime.h" + +namespace lbug { +namespace lance_extension { + +void LanceConnector::connect(const std::string& /*dbPath*/, const std::string& /*catalogName*/, + const std::string& /*schemaName*/, main::ClientContext* context) { + // Creates an in-memory duckdb instance, then loads the lance extension (the + // same engine as lance-duckdb: a DuckDB core extension) and httpfs for + // object-store URIs. + instance = std::make_unique(nullptr); + connection = std::make_unique(*instance); + executeQuery("INSTALL lance;"); + executeQuery("LOAD lance;"); + executeQuery("INSTALL httpfs;"); + executeQuery("LOAD httpfs;"); + initRemoteFSSecrets(context); +} + +std::unique_ptr LanceConnector::newConnection(main::ClientContext* context) { + DASSERT(instance != nullptr); + auto conn = std::make_unique(*instance); + // LOAD is idempotent, and object-store secrets configured by connect() live on the + // shared instance, so every connection inherits them. + auto result = conn->Query("LOAD lance;"); + if (result->HasError()) { + throw common::RuntimeException{ + "Failed to load lance extension on scan connection: " + result->GetError()}; + } + return conn; +} + +std::unique_ptr LanceConnector::executeOn(duckdb::Connection& connection, + const std::string& query) const { + auto result = connection.Query(query); + if (result->HasError()) { + throw common::Exception{result->GetError()}; + } + return result; +} + +std::string LanceConnector::escapeLiteral(const std::string& value) { + std::string out; + out.reserve(value.size()); + for (char c : value) { + if (c == '\'') { + out += "''"; + } else { + out += c; + } + } + return out; +} + +std::string LanceConnector::quoteIdent(const std::string& ident) { + std::string out = "\""; + for (char c : ident) { + if (c == '"') { + out += "\"\""; + } else { + out += c; + } + } + out += "\""; + return out; +} + +std::string LanceConnector::scanAllQuery(const std::string& datasetPath) { + return "SELECT * FROM '" + escapeLiteral(datasetPath) + "'"; +} + +std::string LanceConnector::countRowsQuery(const std::string& datasetPath) { + return "SELECT COUNT(*) FROM '" + escapeLiteral(datasetPath) + "'"; +} + +std::string LanceConnector::schemaQuery(const std::string& datasetPath) { + return scanAllQuery(datasetPath) + " LIMIT 0"; +} + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/function/CMakeLists.txt b/lance/src/function/CMakeLists.txt new file mode 100644 index 00000000..6d9ec8d2 --- /dev/null +++ b/lance/src/function/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(lbug_lance_function + OBJECT + lance_scan.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/connector/duckdb_result_converter.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/connector/duckdb_type_converter.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/function/duckdb_scan.cpp) + +set(LANCE_EXTENSION_OBJECT_FILES + ${LANCE_EXTENSION_OBJECT_FILES} $ + PARENT_SCOPE) diff --git a/lance/src/function/lance_scan.cpp b/lance/src/function/lance_scan.cpp new file mode 100644 index 00000000..bf7291de --- /dev/null +++ b/lance/src/function/lance_scan.cpp @@ -0,0 +1,82 @@ +#include "function/lance_scan.h" + +#include + +namespace lbug { +namespace lance_extension { + +using namespace function; +using namespace common; + +static std::unique_ptr bindFunc(main::ClientContext* context, + const TableFuncBindInput* input) { + // LOAD FROM / COPY callers attach scan metadata; direct CALLs do not. + const auto* scanInput = input->extraInput ? + input->extraInput->constPtrCast() : + nullptr; + auto connector = std::make_shared(); + connector->connect("" /* inMemDB */, "" /* defaultCatalogName */, "" /* defaultSchemaName */, + context); + // DuckDB's lance extension resolves '.lance' through its replacement scan. + std::string query = std::format("SELECT * FROM '{}'", + LanceConnector::escapeLiteral(input->getLiteralVal(0))); + auto result = connector->executeQuery(query + " LIMIT 1"); + std::vector returnTypes; + std::vector returnColumnNames = + scanInput ? scanInput->expectedColumnNames : std::vector{}; + for (auto type : result->GetTypes()) { + returnTypes.push_back( + duckdb_extension::DuckDBTypeConverter::convertDuckDBType(type.ToString())); + } + + if (returnColumnNames.empty()) { + for (auto name : result->GetNames()) { + returnColumnNames.push_back(name.GetIdentifierName()); + } + } + DASSERT(returnTypes.size() == returnColumnNames.size()); + returnColumnNames = + TableFunction::extractYieldVariables(returnColumnNames, input->yieldVariables); + auto columns = input->binder->createVariables(returnColumnNames, returnTypes); + return std::make_unique(std::move(query), connector, + duckdb_extension::DuckDBResultConverter{returnTypes}, columns, context); +} + +std::unique_ptr initLanceScanSharedState( + const TableFuncInitSharedStateInput& input) { + auto lanceScanBindData = input.bindData->constPtrCast(); + auto queryResult = lanceScanBindData->connector->executeQuery(lanceScanBindData->query); + return std::make_unique(std::move(queryResult)); +} + +offset_t tableFunc(const TableFuncInput& input, TableFuncOutput& output) { + auto sharedState = input.sharedState->ptrCast(); + auto lanceScanBindData = input.bindData->constPtrCast(); + std::unique_ptr result; + try { + // Duckdb queryResult.fetch() is not thread safe, we have to acquire a lock there. + std::lock_guard lock{sharedState->mtx}; + result = sharedState->queryResult->Fetch(); + } catch (std::exception& e) { + return 0; + } + if (result == nullptr) { + return 0; + } + lanceScanBindData->converter.convertDuckDBResultToVector(*result, output.dataChunk); + return output.dataChunk.state->getSelVector().getSelSize(); +} + +function_set LanceScanFunction::getFunctionSet() { + function_set functionSet; + auto function = std::make_unique(name, std::vector{LogicalTypeID::STRING}); + function->tableFunc = tableFunc; + function->bindFunc = bindFunc; + function->initSharedStateFunc = initLanceScanSharedState; + function->initLocalStateFunc = TableFunction::initEmptyLocalState; + functionSet.push_back(std::move(function)); + return functionSet; +} + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/include/connector/lance_connector.h b/lance/src/include/connector/lance_connector.h new file mode 100644 index 00000000..6a716b03 --- /dev/null +++ b/lance/src/include/connector/lance_connector.h @@ -0,0 +1,35 @@ +#pragma once + +#include "connector/duckdb_connector.h" + +namespace lbug { +namespace lance_extension { + +// DuckDB-backed connector for Lance datasets. Creates an in-memory DuckDB +// instance with the lance (and httpfs, for object-store URIs) extensions +// loaded, so Lance datasets can be queried with the same engine that powers +// lance-duckdb: `SELECT ... FROM '.lance'` (replacement scan), +// `COPY ... TO ... (FORMAT lance)`, and friends. +class LanceConnector : public duckdb_extension::DuckDBConnector { +public: + void connect(const std::string& dbPath, const std::string& catalogName, + const std::string& schemaName, main::ClientContext* context) override; + + // Opens an additional connection on the shared instance for one scan state. + // DuckDB connections must not run concurrent queries, so every scan state + // owns its connection and issues its range queries without locking. + std::unique_ptr newConnection(main::ClientContext* context); + + std::unique_ptr executeOn(duckdb::Connection& connection, + const std::string& query) const; + + // SQL builders. Identifiers are double-quoted, string literals single-quoted. + static std::string escapeLiteral(const std::string& value); + static std::string quoteIdent(const std::string& ident); + static std::string scanAllQuery(const std::string& datasetPath); + static std::string countRowsQuery(const std::string& datasetPath); + static std::string schemaQuery(const std::string& datasetPath); +}; + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/include/function/lance_scan.h b/lance/src/include/function/lance_scan.h new file mode 100644 index 00000000..94c18642 --- /dev/null +++ b/lance/src/include/function/lance_scan.h @@ -0,0 +1,50 @@ +#pragma once + +#include "binder/binder.h" +#include "connector/connector_factory.h" +#include "connector/duckdb_result_converter.h" +#include "connector/duckdb_type_converter.h" +#include "connector/lance_connector.h" +#include "function/table/table_function.h" + +namespace lbug { +namespace lance_extension { + +// Reads a Lance dataset through DuckDB's lance extension +// (lance-duckdb replacement scan: `SELECT * FROM '.lance'`). +// Registering LANCE_SCAN also enables `LOAD FROM ... (file_format='lance')` +// and `COPY ... FROM ... (file_format='lance')` via the `_SCAN` +// convention. +struct LanceScanFunction { + static constexpr const char* name = "LANCE_SCAN"; + + static function::function_set getFunctionSet(); +}; + +struct LanceScanBindData final : function::ScanFileBindData { + std::string query; + std::shared_ptr connector; + duckdb_extension::DuckDBResultConverter converter; + + LanceScanBindData(std::string query, + std::shared_ptr connector, + duckdb_extension::DuckDBResultConverter converter, binder::expression_vector columns, + main::ClientContext* context) + : function::ScanFileBindData{std::move(columns), 0 /* numRows */, common::FileScanInfo{}, + context}, + query{std::move(query)}, connector{std::move(connector)}, + converter{std::move(converter)} {} + + std::unique_ptr copy() const override { + return std::make_unique(*this); + } +}; + +std::unique_ptr initLanceScanSharedState( + const function::TableFuncInitSharedStateInput& input); + +common::offset_t tableFunc(const function::TableFuncInput& input, + function::TableFuncOutput& output); + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/include/main/lance_extension.h b/lance/src/include/main/lance_extension.h new file mode 100644 index 00000000..fdbbfd94 --- /dev/null +++ b/lance/src/include/main/lance_extension.h @@ -0,0 +1,17 @@ +#pragma once + +#include "extension/extension.h" + +namespace lbug { +namespace lance_extension { + +class LanceExtension final : public extension::Extension { +public: + static constexpr char EXTENSION_NAME[] = "LANCE"; + +public: + static void load(main::ClientContext* context); +}; + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/include/storage/lance_constants.h b/lance/src/include/storage/lance_constants.h new file mode 100644 index 00000000..04c1015f --- /dev/null +++ b/lance/src/include/storage/lance_constants.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include "common/constants.h" + +namespace lbug { +namespace lance_extension { + +// Version and tuning constants for the Lance storage format (v1). +// Mirrors IceDiskConstants, but for Lance datasets instead of Parquet files. +struct LanceConstants { + static constexpr std::string_view V1 = "v1"; + + static constexpr std::string_view CURRENT_VERSION = V1; + + // Contract key for the dataset schema metadata written by the converter. + // v1 readers validate structural compatibility at CREATE TABLE time (see + // LanceUtils::checkCompatibility); strict metadata enforcement arrives + // with the native Lance reader (see docs/lance.md). + static constexpr std::string_view VERSION_METADATA_KEY = "lance_version"; + + static constexpr std::string_view DATASET_SUFFIX = ".lance"; + + // Node-table scan morsel sizing. Morsels are row ranges read with + // LIMIT/OFFSET, which lance-duckdb pushes into the Lance scan, so each + // morsel is one efficient range query. Bounded to keep per-morsel + // materialized DuckDB results small. + static constexpr uint64_t MORSEL_TARGET_ROWS = 8192; + static constexpr uint64_t MAX_MORSELS = 4096; +}; + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/include/storage/lance_node_table.h b/lance/src/include/storage/lance_node_table.h new file mode 100644 index 00000000..353b1a4e --- /dev/null +++ b/lance/src/include/storage/lance_node_table.h @@ -0,0 +1,110 @@ +#pragma once + +#include +#include +#include + +#include "catalog/catalog_entry/node_table_catalog_entry.h" +#include "common/exception/runtime.h" +#include "common/types/internal_id_util.h" +#include "storage/lance_reader.h" +#include "storage/table/columnar_node_table_base.h" + +namespace lbug { +namespace lance_extension { + +// Read-only node table backed by a Lance dataset. Mirrors IceDiskNodeTable: +// one dataset row per node, node offsets are positional row indices, and the +// primary key is a regular property column. The dataset is +// `/nodes_.lance`, or a single `.lance` dataset when storage +// points at one directly. +struct LanceNodeTableScanState final : storage::ColumnarNodeTableScanState { + std::shared_ptr connector; + std::string datasetPath; + std::unique_ptr reader; + // Current morsel: global row range [morselStart, morselStart + morselLimit). + uint64_t morselStart = 0; + uint64_t morselLimit = 0; + // Rows of the current morsel consumed so far (for global offset mapping). + uint64_t morselConsumed = 0; + // Requested column IDs, snapshotted when the reader is built. + std::vector readerColumnIDs; + + LanceNodeTableScanState([[maybe_unused]] storage::MemoryManager& mm, + common::ValueVector* nodeIDVector, std::vector outputVectors, + std::shared_ptr outChunkState) + : storage::ColumnarNodeTableScanState{mm, nodeIDVector, std::move(outputVectors), + std::move(outChunkState)} {} +}; + +struct LanceNodeTableScanSharedState final : storage::ColumnarNodeTableScanSharedState { +private: + std::mutex mtx; + common::node_group_idx_t currentMorselIdx = 0; + common::node_group_idx_t numMorsels = 0; + +public: + void reset(common::node_group_idx_t totalMorsels) { + std::lock_guard lock(mtx); + currentMorselIdx = 0; + numMorsels = totalMorsels; + } + + bool getNextMorsel(common::node_group_idx_t& assignedMorselIdx) { + std::lock_guard lock(mtx); + if (currentMorselIdx < numMorsels) { + assignedMorselIdx = currentMorselIdx++; + return true; + } + return false; + } + + bool getNextMorsel(storage::ColumnarNodeTableScanState*) override { return false; } +}; + +class LanceNodeTable final : public storage::ColumnarNodeTableBase { +public: + LanceNodeTable(const storage::StorageManager* storageManager, + const catalog::NodeTableCatalogEntry* nodeTableEntry, storage::MemoryManager* memoryManager, + main::ClientContext* context = nullptr); + + void initializeScanCoordination(const transaction::Transaction* transaction) override; + + void initScanState(transaction::Transaction* transaction, storage::TableScanState& scanState, + bool resetCachedBoundNodeSelVec = true) const override; + + bool scanInternal(transaction::Transaction* transaction, + storage::TableScanState& scanState) override; + + bool isVisible(const transaction::Transaction* transaction, + common::offset_t offset) const override; + bool isVisibleNoLock(const transaction::Transaction* transaction, + common::offset_t offset) const override; + bool lookupPK(const transaction::Transaction* transaction, common::ValueVector* keyVector, + uint64_t vectorPos, common::offset_t& result) const override; + + const std::string& getDatasetPath() const { return datasetPath; } + // Morsel count for ScanNodeTableSharedState::initialize (via ExternalTableHooks). + common::node_group_idx_t getNumScanMorsels(const transaction::Transaction* transaction) const; + +protected: + std::string getColumnarFormatName() const override { return "lance"; } + common::node_group_idx_t getNumBatches( + const transaction::Transaction* transaction) const override; + common::row_idx_t getTotalRowCount(const transaction::Transaction* transaction) const override; + +private: + std::shared_ptr connector; + std::string datasetPath; + mutable std::mutex schemaMutex; + mutable std::vector datasetSchema; + mutable std::atomic cachedRowCount{common::INVALID_ROW_IDX}; + + void ensureSchemaLoaded(main::ClientContext* context) const; + void checkCompatibility(main::ClientContext* context) const; + void initReaderForMorsel(transaction::Transaction* transaction, + LanceNodeTableScanState& scanState) const; +}; + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/include/storage/lance_reader.h b/lance/src/include/storage/lance_reader.h new file mode 100644 index 00000000..db6843bf --- /dev/null +++ b/lance/src/include/storage/lance_reader.h @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include +#include + +#include "common/data_chunk/data_chunk.h" +#include "connector/duckdb_connector.h" +#include "connector/duckdb_result_converter.h" +#include "connector/duckdb_type_converter.h" +#include "connector/lance_connector.h" + +namespace lbug { +namespace lance_extension { + +// One dataset column: its name, its DuckDB type spelling, and the converted Ladybug type. +struct LanceDatasetColumn { + std::string name; + std::string duckdbType; + common::LogicalType ladybugType; +}; + +// Stateful range reader over one Lance dataset. Issues one DuckDB/SQL query per +// range (`SELECT FROM '' LIMIT/OFFSET`, which lance-duckdb pushes +// into the Lance scan) and decodes result chunks into Ladybug DataChunks with +// DuckDBResultConverter. One reader serves one scan state; ranges are opened +// sequentially (a new range replaces the previous query), so no locking is +// needed. Mirrors the ParquetReader surface used by the icebug-disk tables. +class LanceReader { +public: + // One projected output column: the dataset column to read, the DuckDB CAST + // target aligning it with the catalog type, and the Ladybug output type. + struct ProjectedColumn { + std::string datasetName; + std::string castSpelling; + common::LogicalType ladybugType; + }; + + LanceReader(std::shared_ptr connector, std::string datasetPath, + std::vector projection, main::ClientContext* context); + + const std::vector& getProjection() const { return projection; } + std::vector getProjectionTypes() const { + std::vector types; + types.reserve(projection.size()); + for (auto& col : projection) { + types.push_back(col.ladybugType.copy()); + } + return types; + } + + // Issues a new range query, replacing any active one. limit == UINT64_MAX scans to end. + void openRange(uint64_t offset, uint64_t limit); + // Fetches the next chunk of the active range into a fresh `out` (whose vectors must + // match the projection in order). Returns rows read; 0 means exhausted. + uint64_t next(common::DataChunk& out); + + // Metadata helpers (each opens a short-lived connection). + static std::vector fetchSchema(LanceConnector& connector, + main::ClientContext* context, const std::string& datasetPath); + static uint64_t countRows(LanceConnector& connector, main::ClientContext* context, + const std::string& datasetPath); + + // DuckDB CAST target spelling for a catalog type, so decoded values have the physical + // layout DuckDBResultConverter expects. Throws for STRUCT/MAP/UNION and other types + // with no lossless spelling (see docs/lance.md, supported types). + static std::string toDuckDBCastSpelling(const common::LogicalType& type); + +private: + std::string buildRangeQuery(uint64_t offset, uint64_t limit) const; + + std::shared_ptr connector; + std::string datasetPath; + std::vector projection; + main::ClientContext* context; + std::unique_ptr connection; + std::unique_ptr activeResult; + duckdb_extension::DuckDBResultConverter converter; +}; + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/include/storage/lance_rel_table.h b/lance/src/include/storage/lance_rel_table.h new file mode 100644 index 00000000..2ceffe8a --- /dev/null +++ b/lance/src/include/storage/lance_rel_table.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include + +#include "catalog/catalog_entry/rel_group_catalog_entry.h" +#include "common/exception/runtime.h" +#include "common/types/internal_id_util.h" +#include "storage/lance_reader.h" +#include "storage/table/columnar_rel_table_base.h" +#include "storage/table/ice_disk_rel_table.h" +#include "transaction/transaction.h" + +namespace lbug { +namespace lance_extension { + +enum class LanceRelTableLayout : uint8_t { CSR, FLAT }; + +// Extends IceDiskRelTableScanState (rather than RelTableScanState directly) so a single +// shared scan state can serve lance, icebug-disk, Arrow, and native rel tables in +// multi-rel scans; see ExternalTableHooks::scanStateRank. The Parquet reader members +// inherited from the icebug-disk state stay null; Lance reads go through `reader`. +struct LanceRelTableScanState final : storage::IceDiskRelTableScanState { + std::unique_ptr reader; + // Maps scan-output positions to cached-batch positions (catalog column ID order is + // not assumed to match dataset order; edge properties match by name). + std::vector outputToCached; + // Projection column IDs requested from the dataset, parallel to the reader projection. + // outputToCached[outCol] indexes into this vector; -1 means NULL fill. + std::vector projColumnIDs; + + LanceRelTableScanState(storage::MemoryManager& mm, common::ValueVector* nodeIDVector, + std::vector outputVectors, + std::shared_ptr outChunkState) + : storage::IceDiskRelTableScanState{mm, nodeIDVector, std::move(outputVectors), + std::move(outChunkState)} {} + // setToTable is inherited from IceDiskRelTableScanState: CSR column resolution against + // the (empty) RelTableData scaffolding is safe for Lance tables exactly as for + // icebug-disk tables; the Lance scan reads through `reader` instead. +}; + +class LanceRelTable final : public storage::ColumnarRelTableBase { +public: + LanceRelTable(catalog::RelGroupCatalogEntry* relGroupEntry, common::table_id_t fromTableID, + common::table_id_t toTableID, const storage::StorageManager* storageManager, + storage::MemoryManager* memoryManager, main::ClientContext* context = nullptr); + + void initScanState(transaction::Transaction* transaction, storage::TableScanState& scanState, + bool resetCachedBoundNodeSelVec = true) const override; + + bool scanInternal(transaction::Transaction* transaction, + storage::TableScanState& scanState) override; + + LanceRelTableLayout getLayout() const { return layout; } + const std::string& getIndicesDatasetPath() const { return indicesDatasetPath; } + +protected: + std::string getColumnarFormatName() const override { return "lance"; } + common::row_idx_t getTotalRowCount(const transaction::Transaction* transaction) const override; + common::row_idx_t getActiveBoundNodeCount(const transaction::Transaction* transaction, + common::RelDataDirection direction) const override; + std::vector> getAllDegreeEntries( + const transaction::Transaction* transaction, + common::RelDataDirection direction) const override; + common::row_idx_t getDegreeForOffsetInternal(const transaction::Transaction* transaction, + common::RelDataDirection direction, common::offset_t nodeOffset) const override; + std::vector> getTopKDegreeEntries( + const transaction::Transaction* transaction, common::RelDataDirection direction, + common::idx_t k) const override; + +private: + std::shared_ptr connector; + LanceRelTableLayout layout; + std::string indicesDatasetPath; + std::string indptrDatasetPath; + mutable std::mutex schemaMutex; + mutable std::vector indicesSchema; + mutable std::mutex indptrDataMutex; + mutable std::vector indptrData; + mutable std::atomic indptrDataLoaded{false}; + mutable std::atomic cachedRowCount{common::INVALID_ROW_IDX}; + + void ensureSchemasLoaded(main::ClientContext* context) const; + void checkCompatibility(main::ClientContext* context) const; + void loadIndptrData(transaction::Transaction* transaction) const; + void ensureReader(transaction::Transaction* transaction, + LanceRelTableScanState& scanState) const; + void fillCachedBatch(transaction::Transaction* transaction, + LanceRelTableScanState& scanState) const; + bool scanCSR(transaction::Transaction* transaction, LanceRelTableScanState& scanState); + bool scanFlat(transaction::Transaction* transaction, LanceRelTableScanState& scanState); +}; + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/include/storage/lance_utils.h b/lance/src/include/storage/lance_utils.h new file mode 100644 index 00000000..e41ae52a --- /dev/null +++ b/lance/src/include/storage/lance_utils.h @@ -0,0 +1,67 @@ +#pragma once + +#include + +#include "common/string_utils.h" +#include "storage/lance_constants.h" + +namespace lbug { +namespace lance_extension { + +// Path helpers for the Lance storage layout (v1). Mirrors IceDiskUtils, with +// `.lance` datasets in place of `.parquet` files: +// +// /nodes_
.lance node table dataset +// /indices_
.lance CSR edge targets (+ edge properties) +// /indptr_
.lance CSR row pointers (single integer column) +// .lance single-dataset storage (node table, or FLAT rel table) +struct LanceCSRPaths { + std::string indices; + std::string indptr; +}; + +class LanceUtils { +public: + // Joins a base path with a filename. When base is empty the filename is returned + // as-is (i.e. relative to the current working directory). + static std::string joinPath(const std::string& base, const std::string& part) { + if (base.empty()) { + return part; + } + const char last = base.back(); + if (last == '/' || last == '\\') { + return base + part; + } + return base + "/" + part; + } + + static bool isSingleDatasetPath(const std::string& storage) { + return common::StringUtils::getLower(storage).ends_with(LanceConstants::DATASET_SUFFIX); + } + + // Get the dataset path for a node table. + static std::string constructNodeTablePath(const std::string& dir, const std::string& name) { + if (isSingleDatasetPath(dir)) { + return dir; + } + return joinPath(dir, "nodes_" + name + std::string(LanceConstants::DATASET_SUFFIX)); + } + + // Get the dataset paths for a rel table's CSR files. + static LanceCSRPaths constructCSRPaths(const std::string& dir, const std::string& name) { + return {joinPath(dir, "indices_" + name + std::string(LanceConstants::DATASET_SUFFIX)), + joinPath(dir, "indptr_" + name + std::string(LanceConstants::DATASET_SUFFIX))}; + } + + // Get the dataset path for a flat relationship table. The dataset contains source + // and target node offsets followed by relationship property columns. + static std::string constructFlatRelTablePath(const std::string& dir, const std::string& name) { + if (isSingleDatasetPath(dir)) { + return dir; + } + return joinPath(dir, "rels_" + name + std::string(LanceConstants::DATASET_SUFFIX)); + } +}; + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/installer/CMakeLists.txt b/lance/src/installer/CMakeLists.txt new file mode 100644 index 00000000..de2df4e5 --- /dev/null +++ b/lance/src/installer/CMakeLists.txt @@ -0,0 +1,11 @@ +if (NOT WIN32) + add_library(lance_installer + SHARED + lance_install_func.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/installer/duckdb_installer.cpp) + set_extension_properties(lance_installer lance_installer lance) +endif () + +if (APPLE) + set_apple_dynamic_lookup(lance_installer) +endif () diff --git a/lance/src/installer/lance_install_func.cpp b/lance/src/installer/lance_install_func.cpp new file mode 100644 index 00000000..4315d417 --- /dev/null +++ b/lance/src/installer/lance_install_func.cpp @@ -0,0 +1,16 @@ +#include "installer/duckdb_installer.h" + +extern "C" { +// Because we link against the static library on windows, we implicitly inherit LBUG_STATIC_DEFINE, +// which cancels out any exporting, so we can't use LBUG_API. +#if defined(_WIN32) +#define INIT_EXPORT __declspec(dllexport) +#else +#define INIT_EXPORT __attribute__((visibility("default"))) +#endif +INIT_EXPORT void install(const std::string& repo, lbug::main::ClientContext& context) { + lbug::extension::InstallExtensionInfo info{"lance", repo, false /* forceInstall */}; + lbug::duckdb_extension::DuckDBInstaller installer{info, context}; + installer.install(); +} +} diff --git a/lance/src/main/CMakeLists.txt b/lance/src/main/CMakeLists.txt new file mode 100644 index 00000000..c0e7a7f2 --- /dev/null +++ b/lance/src/main/CMakeLists.txt @@ -0,0 +1,9 @@ +add_library(lance_extension_main + OBJECT + lance_extension.cpp + ${PROJECT_SOURCE_DIR}/extension/duckdb/src/main/duckdb_options.cpp + ${PROJECT_SOURCE_DIR}/extension/httpfs/src/s3fs_config.cpp) + +set(LANCE_EXTENSION_OBJECT_FILES + ${LANCE_EXTENSION_OBJECT_FILES} $ + PARENT_SCOPE) diff --git a/lance/src/main/lance_extension.cpp b/lance/src/main/lance_extension.cpp new file mode 100644 index 00000000..ea031beb --- /dev/null +++ b/lance/src/main/lance_extension.cpp @@ -0,0 +1,77 @@ +#include "main/lance_extension.h" + +#include "function/lance_scan.h" +#include "main/client_context.h" +#include "main/database.h" +#include "main/duckdb_extension.h" +#include "storage/lance_node_table.h" +#include "storage/lance_rel_table.h" +#include "storage/table/external_table_hooks.h" + +namespace lbug { +namespace lance_extension { + +void LanceExtension::load(main::ClientContext* context) { + auto& db = *context->getDatabase(); + extension::ExtensionUtils::addTableFunc(db); + duckdb_extension::DuckdbExtension::loadRemoteFSOptions(context); + // Register Lance-backed table storage for StorageFormat::LANCE. The scan-state + // rank (3) declares that Lance scan states extend the icebug-disk scan state and + // therefore cover every lower-rank backend in shared multi-rel scans. + storage::ExternalTableHooks hooks; + hooks.createNodeTable = [](const storage::StorageManager* storageManager, + const catalog::NodeTableCatalogEntry* nodeTableEntry, + storage::MemoryManager* memoryManager, main::ClientContext* ctx) { + return std::make_unique(storageManager, nodeTableEntry, memoryManager, ctx); + }; + hooks.createRelTable = [](catalog::RelGroupCatalogEntry* relGroupEntry, + common::table_id_t fromTableID, common::table_id_t toTableID, + const storage::StorageManager* storageManager, + storage::MemoryManager* memoryManager, main::ClientContext* ctx) { + return std::make_unique(relGroupEntry, fromTableID, toTableID, + storageManager, memoryManager, ctx); + }; + hooks.createNodeScanState = [](storage::NodeTable* table, common::ValueVector* nodeIDVector, + const std::vector& outVectors, + std::shared_ptr outChunkState, + storage::MemoryManager* memoryManager) { + return std::make_unique(*memoryManager, nodeIDVector, outVectors, + std::move(outChunkState)); + }; + hooks.getNumScanMorsels = [](const transaction::Transaction* transaction, + storage::NodeTable* table) { + return table->ptrCast()->getNumScanMorsels(transaction); + }; + hooks.createRelScanState = [](storage::RelTable* table, common::ValueVector* boundNodeIDVector, + const std::vector& outVectors, + std::shared_ptr outChunkState, + storage::MemoryManager* memoryManager) { + return std::make_unique(*memoryManager, boundNodeIDVector, + outVectors, std::move(outChunkState)); + }; + hooks.scanStateRank = 3; + storage::ExternalTableHookRegistry::registerHooks(common::StorageFormat::LANCE, + std::move(hooks)); +} + +} // namespace lance_extension +} // namespace lbug + +#if defined(BUILD_DYNAMIC_LOAD) +extern "C" { +// Because we link against the static library on windows, we implicitly inherit LBUG_STATIC_DEFINE, +// which cancels out any exporting, so we can't use LBUG_API. +#if defined(_WIN32) +#define INIT_EXPORT __declspec(dllexport) +#else +#define INIT_EXPORT __attribute__((visibility("default"))) +#endif +INIT_EXPORT void init(lbug::main::ClientContext* context) { + lbug::lance_extension::LanceExtension::load(context); +} + +INIT_EXPORT const char* name() { + return lbug::lance_extension::LanceExtension::EXTENSION_NAME; +} +} +#endif diff --git a/lance/src/storage/CMakeLists.txt b/lance/src/storage/CMakeLists.txt new file mode 100644 index 00000000..e9641508 --- /dev/null +++ b/lance/src/storage/CMakeLists.txt @@ -0,0 +1,9 @@ +add_library(lbug_lance_storage + OBJECT + lance_reader.cpp + lance_node_table.cpp + lance_rel_table.cpp) + +set(LANCE_EXTENSION_OBJECT_FILES + ${LANCE_EXTENSION_OBJECT_FILES} $ + PARENT_SCOPE) diff --git a/lance/src/storage/lance_node_table.cpp b/lance/src/storage/lance_node_table.cpp new file mode 100644 index 00000000..afc640a6 --- /dev/null +++ b/lance/src/storage/lance_node_table.cpp @@ -0,0 +1,332 @@ +#include "storage/lance_node_table.h" + +#include +#include + +#include "catalog/catalog_entry/node_table_catalog_entry.h" +#include "common/cast.h" +#include "common/data_chunk/data_chunk.h" +#include "common/data_chunk/sel_vector.h" +#include "common/exception/runtime.h" +#include "common/file_system/local_file_system.h" +#include "common/file_system/virtual_file_system.h" +#include "common/vector/value_vector.h" +#include "main/client_context.h" +#include "storage/buffer_manager/memory_manager.h" +#include "storage/lance_constants.h" +#include "storage/lance_utils.h" +#include "storage/storage_manager.h" +#include "transaction/transaction.h" + +using namespace lbug::catalog; +using namespace lbug::common; +using namespace lbug::storage; +using namespace lbug::transaction; + +namespace lbug { +namespace lance_extension { + +LanceNodeTable::LanceNodeTable(const StorageManager* storageManager, + const NodeTableCatalogEntry* nodeTableEntry, MemoryManager* memoryManager, + main::ClientContext* context) + : ColumnarNodeTableBase{storageManager, nodeTableEntry, memoryManager, + std::make_unique()}, + connector{std::make_shared()} { + connector->connect("" /* inMemDB */, "" /* defaultCatalogName */, "" /* defaultSchemaName */, + context); + const auto& storage = nodeTableEntry->getStorage(); + datasetPath = common::VirtualFileSystem::resolvePath(context, + LanceUtils::constructNodeTablePath(storage, nodeTableEntry->getName())); + // Object-store datasets are validated lazily at scan time (credentials may only be + // configured later); local datasets fail fast here, mirroring icebug-disk. + if (common::LocalFileSystem::isLocalPath(datasetPath)) { + ensureSchemaLoaded(context); + checkCompatibility(context); + } +} + +void LanceNodeTable::ensureSchemaLoaded(main::ClientContext* context) const { + if (!datasetSchema.empty()) { + return; + } + std::lock_guard lock{schemaMutex}; + if (!datasetSchema.empty()) { + return; + } + datasetSchema = LanceReader::fetchSchema(*connector, context, datasetPath); +} + +void LanceNodeTable::checkCompatibility(main::ClientContext* context) const { + // The primary key column must exist: node offsets are positional, but every filter + // and lookup addresses nodes through the PK. + const auto pkName = nodeTableCatalogEntry->getPrimaryKeyName(); + const bool hasPK = std::any_of(datasetSchema.begin(), datasetSchema.end(), + [&](const LanceDatasetColumn& col) { return col.name == pkName; }); + if (!hasPK) { + throw RuntimeException{"Lance dataset '" + datasetPath + "' has no primary key column '" + + pkName + "' for node table '" + nodeTableCatalogEntry->getName() + + "'"}; + } + // Every catalog property present in the dataset must have a mappable type, and the + // full projection must bind (trial query over zero rows validates the CASTs). + std::string selectList; + for (auto i = 0u; i < nodeTableCatalogEntry->getNumProperties(); ++i) { + const auto& prop = nodeTableCatalogEntry->getProperty(i); + auto it = std::find_if(datasetSchema.begin(), datasetSchema.end(), + [&](const LanceDatasetColumn& col) { return col.name == prop.getName(); }); + if (it == datasetSchema.end()) { + continue; // Missing properties read as NULL, mirroring icebug-disk. + } + const auto spelling = LanceReader::toDuckDBCastSpelling(prop.getType()); + if (!selectList.empty()) { + selectList += ", "; + } + selectList += "CAST(" + LanceConnector::quoteIdent(it->name) + " AS " + spelling + ")"; + } + if (!selectList.empty()) { + auto conn = connector->newConnection(context); + connector->executeOn(*conn, "SELECT " + selectList + " FROM '" + + LanceConnector::escapeLiteral(datasetPath) + "' LIMIT 0"); + } +} + +void LanceNodeTable::initializeScanCoordination(const transaction::Transaction* transaction) { + auto sharedState = static_cast(tableScanSharedState.get()); + sharedState->reset(getNumBatches(transaction)); +} + +node_group_idx_t LanceNodeTable::getNumScanMorsels(const Transaction* transaction) const { + return getNumBatches(transaction); +} + +node_group_idx_t LanceNodeTable::getNumBatches(const Transaction* transaction) const { + const auto totalRows = getTotalRowCount(transaction); + if (totalRows == 0) { + return 1; + } + auto morsels = + (totalRows + LanceConstants::MORSEL_TARGET_ROWS - 1) / LanceConstants::MORSEL_TARGET_ROWS; + return static_cast(std::min(morsels, LanceConstants::MAX_MORSELS)); +} + +void LanceNodeTable::initReaderForMorsel(Transaction* transaction, + LanceNodeTableScanState& scanState) const { + auto context = transaction->getClientContext(); + if (!context) { + throw RuntimeException("Invalid client context for Lance node scan initialization"); + } + ensureSchemaLoaded(context); + if (!scanState.reader) { + // First init for this scan state: snapshot the requested columns and build the + // reader projection (dataset column + CAST spelling + catalog type per column). + std::vector projection; + std::vector readerColumnIDs; + for (auto columnID : scanState.columnIDs) { + if (columnID == INVALID_COLUMN_ID || columnID == ROW_IDX_COLUMN_ID) { + continue; + } + // Find the catalog property behind this column ID. + std::string propName; + const LogicalType* propType = nullptr; + for (auto i = 0u; i < nodeTableCatalogEntry->getNumProperties(); ++i) { + if (nodeTableCatalogEntry->getColumnID(i) == columnID) { + propName = nodeTableCatalogEntry->getProperty(i).getName(); + propType = &nodeTableCatalogEntry->getProperty(i).getType(); + break; + } + } + if (propName.empty()) { + continue; + } + auto it = std::find_if(datasetSchema.begin(), datasetSchema.end(), + [&](const LanceDatasetColumn& col) { return col.name == propName; }); + if (it == datasetSchema.end()) { + continue; // Filled with NULL below, mirroring icebug-disk. + } + projection.push_back(LanceReader::ProjectedColumn{it->name, + LanceReader::toDuckDBCastSpelling(*propType), propType->copy()}); + readerColumnIDs.push_back(columnID); + } + if (projection.empty()) { + // No property columns requested: surrogate projection over the first dataset + // column; values are discarded and only the row count is used. + if (datasetSchema.empty()) { + throw RuntimeException{"Lance dataset '" + datasetPath + "' has no columns"}; + } + projection.push_back(LanceReader::ProjectedColumn{datasetSchema[0].name, + LanceReader::toDuckDBCastSpelling(datasetSchema[0].ladybugType), + datasetSchema[0].ladybugType.copy()}); + } + scanState.reader = + std::make_unique(connector, datasetPath, std::move(projection), context); + scanState.readerColumnIDs = std::move(readerColumnIDs); + } + // Pull the next morsel from the shared state (mirrors IceDiskNodeTable: the + // morsel-driven assignment from nextMorsel is discarded in favor of the table's + // own counter). + auto* sharedState = static_cast(tableScanSharedState.get()); + node_group_idx_t assignedMorsel = INVALID_NODE_GROUP_IDX; + if (!sharedState->getNextMorsel(assignedMorsel)) { + scanState.scanCompleted = true; + return; + } + const auto totalRows = static_cast(getTotalRowCount(transaction)); + scanState.morselStart = + static_cast(assignedMorsel) * LanceConstants::MORSEL_TARGET_ROWS; + scanState.morselLimit = totalRows > scanState.morselStart ? + std::min(LanceConstants::MORSEL_TARGET_ROWS, + totalRows - scanState.morselStart) : + 0; + scanState.morselConsumed = 0; + scanState.reader->openRange(scanState.morselStart, scanState.morselLimit); +} + +void LanceNodeTable::initScanState(Transaction* transaction, TableScanState& scanState, + [[maybe_unused]] bool resetCachedBoundNodeSelVec) const { + auto& nodeScanState = scanState.cast(); + nodeScanState.source = TableScanSource::COMMITTED; + auto& lanceScanState = common::dynamic_cast_checked(nodeScanState); + lanceScanState.scanCompleted = false; + lanceScanState.nodeGroupIdx = INVALID_NODE_GROUP_IDX; + initReaderForMorsel(transaction, lanceScanState); +} + +bool LanceNodeTable::scanInternal(Transaction* transaction, TableScanState& scanState) { + auto& lanceScanState = common::dynamic_cast_checked(scanState); + if (lanceScanState.scanCompleted || !lanceScanState.reader) { + return false; + } + auto context = transaction->getClientContext(); + auto memoryManager = MemoryManager::Get(*context); + + // Dataset column index per requested output position (INVALID_COLUMN_ID = NULL fill). + // Surrogate projection (no columns requested): single temp column, values discarded. + const bool surrogate = lanceScanState.readerColumnIDs.empty(); + const auto numProjCols = + surrogate ? 1 : static_cast(lanceScanState.readerColumnIDs.size()); + + while (true) { + scanState.resetOutVectors(); + DataChunk lanceDataChunk(numProjCols, scanState.outState); + // NOTE: projection column types are snapshotted from the reader below. + auto projTypes = lanceScanState.reader->getProjectionTypes(); + for (auto i = 0u; i < numProjCols; ++i) { + lanceDataChunk.insert(i, + std::make_shared(projTypes[i].copy(), memoryManager)); + } + lanceDataChunk.state->getSelVectorUnsafe().setToFiltered(0); + const auto rowsRead = lanceScanState.reader->next(lanceDataChunk); + if (rowsRead == 0) { + lanceScanState.scanCompleted = true; + return false; + } + lanceDataChunk.state->getSelVectorUnsafe().setToUnfiltered(rowsRead); + + const auto startOffset = + static_cast(lanceScanState.morselStart + lanceScanState.morselConsumed); + lanceScanState.morselConsumed += rowsRead; + + NodeTable::applySemiMaskFilter(scanState, startOffset, rowsRead, + lanceDataChunk.state->getSelVectorUnsafe()); + const auto selSize = lanceDataChunk.state->getSelVector().getSelSize(); + if (selSize == 0) { + continue; + } + + const auto selectedPositions = lanceDataChunk.state->getSelVector().getSelectedPositions(); + if (!surrogate) { + // Map requested output columns to projection positions by catalog column ID. + std::vector outputToProj(scanState.outputVectors.size(), INVALID_COLUMN_ID); + for (size_t outCol = 0; outCol < scanState.outputVectors.size(); ++outCol) { + for (size_t projCol = 0; projCol < lanceScanState.readerColumnIDs.size(); + ++projCol) { + if (scanState.columnIDs[outCol] == lanceScanState.readerColumnIDs[projCol]) { + outputToProj[outCol] = projCol; + break; + } + } + } + for (size_t outCol = 0; outCol < scanState.outputVectors.size(); ++outCol) { + auto* dstVector = scanState.outputVectors[outCol]; + if (!dstVector) { + continue; + } + auto projCol = outputToProj[outCol]; + if (projCol == INVALID_COLUMN_ID || + projCol >= lanceDataChunk.getNumValueVectors()) { + for (size_t row = 0; row < selSize; ++row) { + dstVector->setNull(row, true); + } + continue; + } + auto& srcVector = lanceDataChunk.getValueVector(projCol); + size_t row = 0; + for (auto sourceRow : selectedPositions) { + dstVector->copyFromVectorData(row++, &srcVector, sourceRow); + } + } + } + + auto tableID = this->getTableID(); + size_t row = 0; + for (auto sourceRow : selectedPositions) { + auto& nodeID = scanState.nodeIDVector->getValue(row++); + nodeID.tableID = tableID; + nodeID.offset = startOffset + sourceRow; + } + scanState.outState->getSelVectorUnsafe().setToUnfiltered(selSize); + return true; + } +} + +row_idx_t LanceNodeTable::getTotalRowCount(const Transaction* transaction) const { + const auto cached = cachedRowCount.load(std::memory_order_relaxed); + if (cached != INVALID_ROW_IDX) { + return cached; + } + auto context = transaction->getClientContext(); + if (!context) { + return 0; + } + try { + const auto count = + static_cast(LanceReader::countRows(*connector, context, datasetPath)); + cachedRowCount.store(count, std::memory_order_relaxed); + return count; + } catch (const std::exception&) { + return 0; + } +} + +bool LanceNodeTable::isVisible(const transaction::Transaction* transaction, + common::offset_t offset) const { + return offset < getTotalRowCount(transaction); +} + +bool LanceNodeTable::isVisibleNoLock(const transaction::Transaction* transaction, + common::offset_t offset) const { + return offset < getTotalRowCount(transaction); +} + +bool LanceNodeTable::lookupPK(const transaction::Transaction* transaction, + common::ValueVector* keyVector, uint64_t vectorPos, common::offset_t& result) const { + // Same fast path as IceDiskNodeTable: INTERNAL_ID keys resolve directly to offsets. + if (keyVector->dataType.getLogicalTypeID() == common::LogicalTypeID::INTERNAL_ID) { + if (keyVector->isNull(vectorPos)) { + return false; + } + const auto nodeID = keyVector->getValue(vectorPos); + if (nodeID.tableID != tableID || nodeID.offset == common::INVALID_OFFSET) { + return false; + } + if (!isVisibleNoLock(transaction, nodeID.offset)) { + return false; + } + result = nodeID.offset; + return true; + } + return NodeTable::lookupPK(transaction, keyVector, vectorPos, result); +} + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/storage/lance_reader.cpp b/lance/src/storage/lance_reader.cpp new file mode 100644 index 00000000..5a618d66 --- /dev/null +++ b/lance/src/storage/lance_reader.cpp @@ -0,0 +1,176 @@ +#include "storage/lance_reader.h" + +#include "common/exception/binder.h" +#include "common/exception/runtime.h" +#include + +namespace lbug { +namespace lance_extension { + +using namespace common; + +std::string LanceReader::toDuckDBCastSpelling(const LogicalType& type) { + switch (type.getLogicalTypeID()) { + case LogicalTypeID::BOOL: + return "BOOLEAN"; + case LogicalTypeID::INT8: + return "TINYINT"; + case LogicalTypeID::INT16: + return "SMALLINT"; + case LogicalTypeID::INT32: + return "INTEGER"; + case LogicalTypeID::INT64: + return "BIGINT"; + case LogicalTypeID::UINT8: + return "UTINYINT"; + case LogicalTypeID::UINT16: + return "USMALLINT"; + case LogicalTypeID::UINT32: + return "UINTEGER"; + case LogicalTypeID::UINT64: + return "UBIGINT"; + case LogicalTypeID::INT128: + return "HUGEINT"; + case LogicalTypeID::UINT128: + return "UHUGEINT"; + case LogicalTypeID::FLOAT: + return "FLOAT"; + case LogicalTypeID::DOUBLE: + return "DOUBLE"; + case LogicalTypeID::STRING: + return "VARCHAR"; + case LogicalTypeID::BLOB: + return "BLOB"; + case LogicalTypeID::DATE: + return "DATE"; + case LogicalTypeID::TIMESTAMP: + return "TIMESTAMP"; + case LogicalTypeID::TIMESTAMP_SEC: + return "TIMESTAMP_S"; + case LogicalTypeID::TIMESTAMP_MS: + return "TIMESTAMP_MS"; + case LogicalTypeID::TIMESTAMP_NS: + return "TIMESTAMP_NS"; + case LogicalTypeID::TIMESTAMP_TZ: + return "TIMESTAMPTZ"; + case LogicalTypeID::INTERVAL: + return "INTERVAL"; + case LogicalTypeID::UUID: + return "UUID"; + case LogicalTypeID::DECIMAL: + return std::format("DECIMAL({},{})", DecimalType::getPrecision(type), + DecimalType::getScale(type)); + case LogicalTypeID::LIST: + return toDuckDBCastSpelling(ListType::getChildType(type)) + "[]"; + case LogicalTypeID::ARRAY: + return std::format("{}[{}]", toDuckDBCastSpelling(ArrayType::getChildType(type)), + ArrayType::getNumElements(type)); + case LogicalTypeID::STRUCT: + case LogicalTypeID::MAP: + case LogicalTypeID::UNION: + throw BinderException{std::format( + "Lance-backed tables do not support {} columns in v1. Flatten nested objects into " + "top-level columns (see docs/lance.md).", + LogicalTypeUtils::toString(type.getLogicalTypeID()))}; + default: + throw BinderException{std::format( + "Lance-backed tables do not support column type {} in v1.", type.toString())}; + } +} + +LanceReader::LanceReader(std::shared_ptr connector_, std::string datasetPath_, + std::vector projection_, main::ClientContext* context_) + : connector{std::move(connector_)}, datasetPath{std::move(datasetPath_)}, + projection{std::move(projection_)}, context{context_}, converter{[this] { + std::vector types; + types.reserve(projection.size()); + for (auto& col : projection) { + types.push_back(col.ladybugType.copy()); + } + return types; + }()} {} + +std::string LanceReader::buildRangeQuery(uint64_t offset, uint64_t limit) const { + std::string selectList; + if (projection.empty()) { + // No property columns requested (e.g. COUNT over node IDs, which are positional): + // read one arbitrary column and discard the values; only the row count matters. + // The caller substitutes a surrogate projection (see LanceNodeTable::scanInternal). + throw RuntimeException{"LanceReader requires a non-empty projection"}; + } + for (auto i = 0u; i < projection.size(); ++i) { + if (i > 0) { + selectList += ", "; + } + selectList += std::format("CAST({} AS {})", + LanceConnector::quoteIdent(projection[i].datasetName), projection[i].castSpelling); + } + auto query = + std::format("SELECT {} FROM '{}'", selectList, LanceConnector::escapeLiteral(datasetPath)); + if (offset > 0) { + query += std::format(" OFFSET {}", offset); + } + if (limit != UINT64_MAX) { + query += std::format(" LIMIT {}", limit); + } + return query; +} + +void LanceReader::openRange(uint64_t offset, uint64_t limit) { + if (!connection) { + connection = connector->newConnection(context); + } + activeResult = connector->executeOn(*connection, buildRangeQuery(offset, limit)); +} + +uint64_t LanceReader::next(DataChunk& out) { + if (!activeResult) { + return 0; + } + std::unique_ptr chunk; + try { + chunk = activeResult->Fetch(); + } catch (std::exception& e) { + throw RuntimeException{ + std::string{"Lance scan failed for '"} + datasetPath + "': " + e.what()}; + } + if (!chunk || chunk->size() == 0) { + activeResult.reset(); + return 0; + } + converter.convertDuckDBResultToVector(*chunk, out); + return out.state->getSelVector().getSelSize(); +} + +std::vector LanceReader::fetchSchema(LanceConnector& connector, + main::ClientContext* context, const std::string& datasetPath) { + auto conn = connector.newConnection(context); + auto result = connector.executeOn(*conn, LanceConnector::schemaQuery(datasetPath)); + const auto& types = result->GetTypes(); + const auto& names = result->GetNames(); + if (types.size() != names.size()) { + throw RuntimeException{"Failed to read Lance dataset schema for '" + datasetPath + "'"}; + } + std::vector schema; + for (auto i = 0u; i < types.size(); ++i) { + const auto typeStr = types[i].ToString(); + schema.push_back(LanceDatasetColumn{names[i].GetIdentifierName(), typeStr, + duckdb_extension::DuckDBTypeConverter::convertDuckDBType(typeStr)}); + } + return schema; +} + +uint64_t LanceReader::countRows(LanceConnector& connector, main::ClientContext* context, + const std::string& datasetPath) { + auto conn = connector.newConnection(context); + auto result = connector.executeOn(*conn, LanceConnector::countRowsQuery(datasetPath)); + auto chunk = result->Fetch(); + if (!chunk || chunk->size() == 0) { + throw RuntimeException{"Failed to count rows of Lance dataset '" + datasetPath + "'"}; + } + // COUNT(*) returns BIGINT. + return static_cast(chunk->GetValue(0, 0).GetValue()); +} + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/storage/lance_rel_table.cpp b/lance/src/storage/lance_rel_table.cpp new file mode 100644 index 00000000..d3f033c9 --- /dev/null +++ b/lance/src/storage/lance_rel_table.cpp @@ -0,0 +1,658 @@ +#include "storage/lance_rel_table.h" + +#include +#include +#include +#include +#include + +#include "catalog/catalog_entry/rel_group_catalog_entry.h" +#include "common/assert.h" +#include "common/cast.h" +#include "common/data_chunk/data_chunk.h" +#include "common/data_chunk/sel_vector.h" +#include "common/exception/runtime.h" +#include "common/file_system/local_file_system.h" +#include "common/file_system/virtual_file_system.h" +#include "common/string_utils.h" +#include "common/vector/value_vector.h" +#include "main/client_context.h" +#include "storage/buffer_manager/memory_manager.h" +#include "storage/lance_utils.h" +#include "storage/storage_manager.h" +#include "storage/table/csr_node_group.h" +#include "transaction/transaction.h" + +using namespace lbug::catalog; +using namespace lbug::common; +using namespace lbug::storage; +using namespace lbug::transaction; + +namespace lbug { +namespace lance_extension { + +LanceRelTable::LanceRelTable(RelGroupCatalogEntry* relGroupEntry, table_id_t fromTableID, + table_id_t toTableID, const StorageManager* storageManager, MemoryManager* memoryManager, + main::ClientContext* context) + : ColumnarRelTableBase{relGroupEntry, fromTableID, toTableID, storageManager, memoryManager}, + connector{std::make_shared()}, layout{LanceRelTableLayout::CSR} { + connector->connect("" /* inMemDB */, "" /* defaultCatalogName */, "" /* defaultSchemaName */, + context); + const auto& storage = relGroupEntry->getStorage(); + if (LanceUtils::isSingleDatasetPath(storage)) { + layout = LanceRelTableLayout::FLAT; + indicesDatasetPath = VirtualFileSystem::resolvePath(context, storage); + } else { + auto paths = LanceUtils::constructCSRPaths(storage, relGroupEntry->getName()); + indicesDatasetPath = VirtualFileSystem::resolvePath(context, paths.indices); + indptrDatasetPath = VirtualFileSystem::resolvePath(context, paths.indptr); + } + // Object-store datasets are validated lazily at scan time; local datasets fail fast + // here, mirroring icebug-disk. + if (common::LocalFileSystem::isLocalPath(indicesDatasetPath)) { + ensureSchemasLoaded(context); + checkCompatibility(context); + } +} + +void LanceRelTable::ensureSchemasLoaded(main::ClientContext* context) const { + if (!indicesSchema.empty()) { + return; + } + std::lock_guard lock{schemaMutex}; + if (!indicesSchema.empty()) { + return; + } + indicesSchema = LanceReader::fetchSchema(*connector, context, indicesDatasetPath); +} + +void LanceRelTable::checkCompatibility(main::ClientContext* context) const { + if (indicesSchema.empty()) { + throw RuntimeException{"Lance dataset '" + indicesDatasetPath + "' has no columns"}; + } + std::vector trial; + if (layout == LanceRelTableLayout::FLAT) { + if (indicesSchema.size() < 2) { + throw RuntimeException{"Flat Lance rel dataset '" + indicesDatasetPath + + "' requires source and target offset columns"}; + } + trial.push_back( + LanceReader::ProjectedColumn{indicesSchema[0].name, "BIGINT", LogicalType::INT64()}); + trial.push_back( + LanceReader::ProjectedColumn{indicesSchema[1].name, "BIGINT", LogicalType::INT64()}); + } else { + // Column 0 is the destination node offset. + trial.push_back( + LanceReader::ProjectedColumn{indicesSchema[0].name, "BIGINT", LogicalType::INT64()}); + auto indptrSchema = LanceReader::fetchSchema(*connector, context, indptrDatasetPath); + if (indptrSchema.empty()) { + throw RuntimeException{ + "Lance indptr dataset '" + indptrDatasetPath + "' has no columns"}; + } + auto conn = connector->newConnection(context); + connector->executeOn(*conn, + "SELECT CAST(" + LanceConnector::quoteIdent(indptrSchema[0].name) + + " AS BIGINT) FROM '" + LanceConnector::escapeLiteral(indptrDatasetPath) + + "' LIMIT 0"); + } + // Edge properties match by name; every present property must have a mappable type. + // Missing properties read as NULL, mirroring icebug-disk. + for (auto i = 0u; i < relGroupEntry->getNumProperties(); ++i) { + const auto& prop = relGroupEntry->getProperty(i); + if (prop.getType().getLogicalTypeID() == LogicalTypeID::INTERNAL_ID) { + continue; // Rel ID is synthesized from the row index, not stored. + } + auto it = std::find_if(indicesSchema.begin(), indicesSchema.end(), + [&](const LanceDatasetColumn& col) { return col.name == prop.getName(); }); + if (it == indicesSchema.end()) { + continue; + } + trial.push_back(LanceReader::ProjectedColumn{it->name, + LanceReader::toDuckDBCastSpelling(prop.getType()), prop.getType().copy()}); + } + // Trial query over zero rows validates that every CAST binds. + std::string selectList; + for (auto& col : trial) { + if (!selectList.empty()) { + selectList += ", "; + } + selectList += + "CAST(" + LanceConnector::quoteIdent(col.datasetName) + " AS " + col.castSpelling + ")"; + } + auto conn = connector->newConnection(context); + connector->executeOn(*conn, "SELECT " + selectList + " FROM '" + + LanceConnector::escapeLiteral(indicesDatasetPath) + + "' LIMIT 0"); +} + +void LanceRelTable::ensureReader(Transaction* transaction, + LanceRelTableScanState& scanState) const { + if (scanState.reader) { + return; + } + auto context = transaction->getClientContext(); + if (!context) { + throw RuntimeException("Invalid client context for Lance rel scan initialization"); + } + ensureSchemasLoaded(context); + // Projection: endpoint offset column(s) first, then edge properties present in the + // dataset (by name), in catalog column order. outputToCached maps each requested + // output position to its cached-batch position (-1 = NULL fill). + if (indicesSchema.empty()) { + throw RuntimeException{"Lance dataset '" + indicesDatasetPath + "' has no columns"}; + } + std::vector projection; + std::vector projColumnIDs; + if (layout == LanceRelTableLayout::FLAT) { + if (indicesSchema.size() < 2) { + throw RuntimeException{"Flat Lance rel dataset '" + indicesDatasetPath + + "' requires source and target offset columns"}; + } + projection.push_back( + LanceReader::ProjectedColumn{indicesSchema[0].name, "BIGINT", LogicalType::INT64()}); + projColumnIDs.push_back(INVALID_COLUMN_ID); + projection.push_back( + LanceReader::ProjectedColumn{indicesSchema[1].name, "BIGINT", LogicalType::INT64()}); + projColumnIDs.push_back(INVALID_COLUMN_ID); + } else { + projection.push_back( + LanceReader::ProjectedColumn{indicesSchema[0].name, "BIGINT", LogicalType::INT64()}); + projColumnIDs.push_back(INVALID_COLUMN_ID); + } + for (auto i = 0u; i < relGroupEntry->getNumProperties(); ++i) { + const auto& prop = relGroupEntry->getProperty(i); + if (prop.getType().getLogicalTypeID() == LogicalTypeID::INTERNAL_ID) { + continue; + } + auto it = std::find_if(indicesSchema.begin(), indicesSchema.end(), + [&](const LanceDatasetColumn& col) { return col.name == prop.getName(); }); + if (it == indicesSchema.end()) { + continue; + } + projection.push_back(LanceReader::ProjectedColumn{it->name, + LanceReader::toDuckDBCastSpelling(prop.getType()), prop.getType().copy()}); + projColumnIDs.push_back(relGroupEntry->getColumnID(i)); + } + scanState.reader = std::make_unique(connector, indicesDatasetPath, + std::move(projection), context); + scanState.projColumnIDs = std::move(projColumnIDs); + scanState.outputToCached.assign(scanState.columnIDs.size(), -1); + for (size_t outCol = 0; outCol < scanState.columnIDs.size(); ++outCol) { + const auto colID = scanState.columnIDs[outCol]; + if (colID == NBR_ID_COLUMN_ID) { + scanState.outputToCached[outCol] = 0; + continue; + } + if (colID == INVALID_COLUMN_ID || colID == ROW_IDX_COLUMN_ID || colID == REL_ID_COLUMN_ID) { + continue; + } + for (size_t projCol = 0; projCol < scanState.projColumnIDs.size(); ++projCol) { + if (scanState.projColumnIDs[projCol] == colID) { + scanState.outputToCached[outCol] = static_cast(projCol); + break; + } + } + } +} + +void LanceRelTable::fillCachedBatch(Transaction* transaction, + LanceRelTableScanState& scanState) const { + auto projTypes = scanState.reader->getProjectionTypes(); + scanState.cachedBatchData = std::make_unique(projTypes.size()); + auto context = transaction->getClientContext(); + auto memoryManager = MemoryManager::Get(*context); + for (auto i = 0u; i < projTypes.size(); ++i) { + scanState.cachedBatchData->insert(i, + std::make_shared(projTypes[i].copy(), memoryManager)); + } + scanState.cachedBatchData->state->getSelVectorUnsafe().setToFiltered(0); + scanState.reader->next(*scanState.cachedBatchData); +} + +void LanceRelTable::initScanState(Transaction* transaction, TableScanState& scanState, + bool resetCachedBoundNodeSelVec) const { + auto& relScanState = scanState.cast(); + relScanState.source = TableScanSource::COMMITTED; + relScanState.nodeGroup = nullptr; + relScanState.nodeGroupIdx = INVALID_NODE_GROUP_IDX; + + if (resetCachedBoundNodeSelVec) { + if (relScanState.nodeIDVector->state->getSelVector().isUnfiltered()) { + relScanState.cachedBoundNodeSelVector.setToUnfiltered(); + } else { + relScanState.cachedBoundNodeSelVector.setToFiltered(); + memcpy(relScanState.cachedBoundNodeSelVector.getMutableBuffer().data(), + relScanState.nodeIDVector->state->getSelVector().getMutableBuffer().data(), + relScanState.nodeIDVector->state->getSelVector().getSelSize() * + sizeof(common::sel_t)); + } + relScanState.cachedBoundNodeSelVector.setSelSize( + relScanState.nodeIDVector->state->getSelVector().getSelSize()); + } + + auto& lanceScanState = common::dynamic_cast_checked(relScanState); + ensureReader(transaction, lanceScanState); + + auto numRows = UINT64_MAX; + auto startRow = 0u; + if (layout == LanceRelTableLayout::CSR) { + loadIndptrData(transaction); + } + + // Create a set of bound node IDs for fast lookup. + std::unordered_map boundNodeOffsets; + for (size_t i = 0; i < lanceScanState.cachedBoundNodeSelVector.getSelSize(); ++i) { + common::sel_t boundNodeIdx = lanceScanState.cachedBoundNodeSelVector[i]; + const auto boundNodeID = lanceScanState.nodeIDVector->getValue(boundNodeIdx); + boundNodeOffsets.insert({boundNodeID.offset, boundNodeIdx}); + } + lanceScanState.reset(std::move(boundNodeOffsets)); + + // Range-limit the indices scan to the dataset rows that can contain edges of this + // bound-node batch (CSR is sorted by source node). FWD only; BWD bound nodes are + // targets scattered through the file. Mirrors IceDiskRelTable. + bool rangeLimited = false; + if (layout == LanceRelTableLayout::CSR && lanceScanState.direction == RelDataDirection::FWD && + !indptrData.empty() && !lanceScanState.boundNodeOffsets.empty()) { + common::offset_t minNode = std::numeric_limits::max(); + common::offset_t maxNode = 0; + for (auto& entry : lanceScanState.boundNodeOffsets) { + minNode = std::min(minNode, entry.first); + maxNode = std::max(maxNode, entry.first); + } + if ((uint64_t)minNode + 1 < indptrData.size()) { + auto endIdx = std::min((uint64_t)maxNode + 1, indptrData.size() - 1); + startRow = (uint64_t)indptrData[minNode]; + numRows = (uint64_t)indptrData[endIdx] - startRow; + rangeLimited = true; + } + } + lanceScanState.currentBatchStartOffset = static_cast(startRow); + lanceScanState.currentLocalRowIdx = 0; + lanceScanState.reader->openRange(startRow, numRows); + + // Re-anchor the monotonic CSR source-node cursor to the node containing the first + // scanned row. Bound-node batches are not guaranteed to arrive in increasing offset + // order, so the cursor is repositioned per batch. Mirrors IceDiskRelTable. + if (layout == LanceRelTableLayout::CSR && !indptrData.empty() && + lanceScanState.currentBatchStartOffset < indptrData.back()) { + lanceScanState.csrSrcNodeIdx = + findSourceNodeForRowInternal(lanceScanState.currentBatchStartOffset, indptrData); + } +} + +void LanceRelTable::loadIndptrData(Transaction* transaction) const { + if (indptrDatasetPath.empty() || indptrDataLoaded.load(std::memory_order_acquire)) { + return; + } + std::unique_lock lock(indptrDataMutex); + if (indptrDataLoaded.load(std::memory_order_relaxed)) { + return; + } + { + auto context = transaction->getClientContext(); + ensureSchemasLoaded(context); + auto indptrSchema = LanceReader::fetchSchema(*connector, context, indptrDatasetPath); + if (indptrSchema.empty()) { + throw RuntimeException("Indptr Lance dataset has no columns"); + } + std::vector projection; + projection.push_back( + LanceReader::ProjectedColumn{indptrSchema[0].name, "BIGINT", LogicalType::INT64()}); + LanceReader reader(connector, indptrDatasetPath, std::move(projection), context); + reader.openRange(0, UINT64_MAX); + auto memoryManager = MemoryManager::Get(*context); + DataChunk chunk(1); + chunk.insert(0, std::make_shared(LogicalType::INT64(), memoryManager)); + while (reader.next(chunk) > 0) { + auto selSize = chunk.state->getSelVector().getSelSize(); + for (size_t i = 0; i < selSize; ++i) { + indptrData.push_back(chunk.getValueVector(0).getValue(i)); + } + chunk.state->getSelVectorUnsafe().setToFiltered(0); + } + indptrDataLoaded.store(true, std::memory_order_release); + } +} + +bool LanceRelTable::scanInternal(Transaction* transaction, TableScanState& scanState) { + auto& lanceScanState = common::dynamic_cast_checked(scanState); + if (layout == LanceRelTableLayout::FLAT) { + return scanFlat(transaction, lanceScanState); + } + return scanCSR(transaction, lanceScanState); +} + +bool LanceRelTable::scanCSR(Transaction* transaction, LanceRelTableScanState& lanceScanState) { + lanceScanState.resetOutVectors(); + + if (lanceScanState.boundNodeOffsets.empty()) { + lanceScanState.outState->getSelVectorUnsafe().setToFiltered(0); + return false; + } + + loadIndptrData(transaction); + + const auto isFwd = lanceScanState.direction != RelDataDirection::BWD; + uint64_t totalRowsCollected = 0; + const uint64_t maxRowsPerCall = DEFAULT_VECTOR_CAPACITY; + auto activeBoundSelPos = INVALID_SEL; + auto activeBoundOffset = INVALID_OFFSET; + auto hasActiveBound = false; + auto differentBoundNodeEncountered = false; + + while (totalRowsCollected < maxRowsPerCall) { + if (!lanceScanState.cachedBatchData || + lanceScanState.currentLocalRowIdx == + lanceScanState.cachedBatchData->state->getSelVector().getSelSize()) { + lanceScanState.currentBatchStartOffset += lanceScanState.currentLocalRowIdx; + lanceScanState.currentLocalRowIdx = 0; + fillCachedBatch(transaction, lanceScanState); + } + + auto selSize = lanceScanState.cachedBatchData->state->getSelVector().getSelSize(); + if (selSize == 0) { + break; + } + + for (; lanceScanState.currentLocalRowIdx < selSize && totalRowsCollected < maxRowsPerCall; + ++lanceScanState.currentLocalRowIdx) { + const auto currentGlobalRowIdx = + lanceScanState.currentBatchStartOffset + lanceScanState.currentLocalRowIdx; + if (indptrData.empty()) { + continue; + } + while (lanceScanState.csrSrcNodeIdx + 1 < indptrData.size() && + indptrData[lanceScanState.csrSrcNodeIdx + 1] <= currentGlobalRowIdx) { + ++lanceScanState.csrSrcNodeIdx; + } + const auto sourceNodeOffset = lanceScanState.csrSrcNodeIdx; + + // Cached column 0 is the destination node offset. + const auto dstOffset = + lanceScanState.cachedBatchData->getValueVector(0).getValue( + lanceScanState.currentLocalRowIdx); + const auto boundOffset = isFwd ? sourceNodeOffset : dstOffset; + if (lanceScanState.boundNodeOffsets.find(boundOffset) == + lanceScanState.boundNodeOffsets.end()) { + continue; + } + + if (!hasActiveBound) { + hasActiveBound = true; + activeBoundOffset = boundOffset; + activeBoundSelPos = lanceScanState.boundNodeOffsets.at(boundOffset); + } else if (boundOffset != activeBoundOffset) { + differentBoundNodeEncountered = true; + break; + } + + const auto nbrOffset = isFwd ? dstOffset : sourceNodeOffset; + const auto nbrTableID = isFwd ? getToNodeTableID() : getFromNodeTableID(); + auto nbrNodeID = internalID_t(nbrOffset, nbrTableID); + + if (!lanceScanState.outputVectors.empty()) { + lanceScanState.outputVectors[0]->setValue(totalRowsCollected, nbrNodeID); + } + + for (uint64_t outCol = 1; outCol < lanceScanState.outputVectors.size(); ++outCol) { + if (outCol >= lanceScanState.columnIDs.size()) { + continue; + } + const auto colID = lanceScanState.columnIDs[outCol]; + if (colID == INVALID_COLUMN_ID || colID == ROW_IDX_COLUMN_ID || + colID == NBR_ID_COLUMN_ID) { + continue; + } + if (colID == REL_ID_COLUMN_ID) { + // REL_ID is not stored; synthesize from the global row index. + lanceScanState.outputVectors[outCol]->setValue(totalRowsCollected, + internalID_t{currentGlobalRowIdx, getTableID()}); + continue; + } + const auto cachedCol = outCol < lanceScanState.outputToCached.size() ? + lanceScanState.outputToCached[outCol] : + -1; + if (cachedCol < 0 || + (uint64_t)cachedCol >= lanceScanState.cachedBatchData->getNumValueVectors()) { + // Property absent from the dataset reads as NULL. + lanceScanState.outputVectors[outCol]->setNull(totalRowsCollected, true); + continue; + } + lanceScanState.outputVectors[outCol]->copyFromVectorData(totalRowsCollected, + &lanceScanState.cachedBatchData->getValueVector(cachedCol), + lanceScanState.currentLocalRowIdx); + } + + totalRowsCollected++; + } + + if (differentBoundNodeEncountered) { + break; + } + } + + if (totalRowsCollected > 0) { + auto& selVector = lanceScanState.outState->getSelVectorUnsafe(); + selVector.setToUnfiltered(totalRowsCollected); + lanceScanState.setNodeIDVectorToFlat(activeBoundSelPos); + return true; + } + lanceScanState.outState->getSelVectorUnsafe().setToFiltered(0); + return false; +} + +bool LanceRelTable::scanFlat(Transaction* transaction, LanceRelTableScanState& lanceScanState) { + lanceScanState.resetOutVectors(); + + if (lanceScanState.boundNodeOffsets.empty()) { + lanceScanState.outState->getSelVectorUnsafe().setToFiltered(0); + return false; + } + + const auto isFwd = lanceScanState.direction != RelDataDirection::BWD; + uint64_t totalRowsCollected = 0; + const uint64_t maxRowsPerCall = DEFAULT_VECTOR_CAPACITY; + auto activeBoundSelPos = INVALID_SEL; + auto activeBoundOffset = INVALID_OFFSET; + auto hasActiveBound = false; + auto differentBoundNodeEncountered = false; + + while (totalRowsCollected < maxRowsPerCall) { + if (!lanceScanState.cachedBatchData || + lanceScanState.currentLocalRowIdx == + lanceScanState.cachedBatchData->state->getSelVector().getSelSize()) { + lanceScanState.currentBatchStartOffset += lanceScanState.currentLocalRowIdx; + lanceScanState.currentLocalRowIdx = 0; + fillCachedBatch(transaction, lanceScanState); + } + + auto selSize = lanceScanState.cachedBatchData->state->getSelVector().getSelSize(); + if (selSize == 0) { + break; + } + + for (; lanceScanState.currentLocalRowIdx < selSize && totalRowsCollected < maxRowsPerCall; + ++lanceScanState.currentLocalRowIdx) { + if (lanceScanState.cachedBatchData->getNumValueVectors() < 2) { + throw RuntimeException("Flat Lance relationship dataset requires source and " + "target offset columns"); + } + const auto currentGlobalRowIdx = + lanceScanState.currentBatchStartOffset + lanceScanState.currentLocalRowIdx; + const auto srcOffset = + lanceScanState.cachedBatchData->getValueVector(0).getValue( + lanceScanState.currentLocalRowIdx); + const auto dstOffset = + lanceScanState.cachedBatchData->getValueVector(1).getValue( + lanceScanState.currentLocalRowIdx); + const auto boundOffset = isFwd ? srcOffset : dstOffset; + auto boundIt = lanceScanState.boundNodeOffsets.find(boundOffset); + if (boundIt == lanceScanState.boundNodeOffsets.end()) { + continue; + } + + if (!hasActiveBound) { + hasActiveBound = true; + activeBoundOffset = boundOffset; + activeBoundSelPos = boundIt->second; + } else if (boundOffset != activeBoundOffset) { + differentBoundNodeEncountered = true; + break; + } + + const auto nbrOffset = isFwd ? dstOffset : srcOffset; + const auto nbrTableID = isFwd ? getToNodeTableID() : getFromNodeTableID(); + if (!lanceScanState.outputVectors.empty()) { + lanceScanState.outputVectors[0]->setValue(totalRowsCollected, + internalID_t(nbrOffset, nbrTableID)); + } + + for (uint64_t outCol = 1; outCol < lanceScanState.outputVectors.size(); ++outCol) { + if (outCol >= lanceScanState.columnIDs.size()) { + continue; + } + const auto colID = lanceScanState.columnIDs[outCol]; + if (colID == INVALID_COLUMN_ID || colID == ROW_IDX_COLUMN_ID || + colID == NBR_ID_COLUMN_ID) { + continue; + } + if (colID == REL_ID_COLUMN_ID) { + lanceScanState.outputVectors[outCol]->setValue(totalRowsCollected, + internalID_t{currentGlobalRowIdx, getTableID()}); + continue; + } + const auto cachedCol = outCol < lanceScanState.outputToCached.size() ? + lanceScanState.outputToCached[outCol] : + -1; + if (cachedCol < 0 || + (uint64_t)cachedCol >= lanceScanState.cachedBatchData->getNumValueVectors()) { + lanceScanState.outputVectors[outCol]->setNull(totalRowsCollected, true); + continue; + } + lanceScanState.outputVectors[outCol]->copyFromVectorData(totalRowsCollected, + &lanceScanState.cachedBatchData->getValueVector(cachedCol), + lanceScanState.currentLocalRowIdx); + } + + totalRowsCollected++; + } + + if (differentBoundNodeEncountered) { + break; + } + } + + if (totalRowsCollected > 0) { + auto& selVector = lanceScanState.outState->getSelVectorUnsafe(); + selVector.setToUnfiltered(totalRowsCollected); + lanceScanState.setNodeIDVectorToFlat(activeBoundSelPos); + return true; + } + lanceScanState.outState->getSelVectorUnsafe().setToFiltered(0); + return false; +} + +row_idx_t LanceRelTable::getTotalRowCount(const Transaction* transaction) const { + const auto cached = cachedRowCount.load(std::memory_order_relaxed); + if (cached != INVALID_ROW_IDX) { + return cached; + } + auto context = transaction->getClientContext(); + if (!context) { + return 0; + } + try { + const auto count = + static_cast(LanceReader::countRows(*connector, context, indicesDatasetPath)); + cachedRowCount.store(count, std::memory_order_relaxed); + return count; + } catch (const std::exception&) { + return 0; + } +} + +row_idx_t LanceRelTable::getActiveBoundNodeCount(const Transaction* transaction, + RelDataDirection direction) const { + if (layout != LanceRelTableLayout::CSR || direction == RelDataDirection::BWD) { + return 0; + } + loadIndptrData(const_cast(transaction)); + row_idx_t result = 0; + for (offset_t i = 0; i + 1 < indptrData.size(); ++i) { + result += indptrData[i + 1] > indptrData[i]; + } + return result; +} + +std::vector> LanceRelTable::getAllDegreeEntries( + const Transaction* transaction, RelDataDirection direction) const { + if (layout != LanceRelTableLayout::CSR || direction == RelDataDirection::BWD) { + return {}; + } + loadIndptrData(const_cast(transaction)); + std::vector> result; + result.reserve(indptrData.size()); + for (offset_t i = 0; i + 1 < indptrData.size(); ++i) { + if (indptrData[i + 1] <= indptrData[i]) { + continue; + } + result.emplace_back(i, indptrData[i + 1] - indptrData[i]); + } + return result; +} + +row_idx_t LanceRelTable::getDegreeForOffsetInternal(const Transaction* transaction, + RelDataDirection direction, offset_t nodeOffset) const { + if (layout != LanceRelTableLayout::CSR || direction == RelDataDirection::BWD) { + return 0; + } + loadIndptrData(const_cast(transaction)); + if (nodeOffset + 1 >= indptrData.size()) { + return 0; + } + if (indptrData[nodeOffset + 1] <= indptrData[nodeOffset]) { + return 0; + } + return indptrData[nodeOffset + 1] - indptrData[nodeOffset]; +} + +std::vector> LanceRelTable::getTopKDegreeEntries( + const Transaction* transaction, RelDataDirection direction, common::idx_t k) const { + if (layout != LanceRelTableLayout::CSR || direction == RelDataDirection::BWD || k == 0) { + return {}; + } + loadIndptrData(const_cast(transaction)); + using degree_entry_t = std::pair; + auto better = [](const degree_entry_t& a, const degree_entry_t& b) { + return a.second > b.second || (a.second == b.second && a.first < b.first); + }; + auto worseForHeap = [better](const degree_entry_t& a, const degree_entry_t& b) { + return better(a, b); + }; + std::priority_queue, decltype(worseForHeap)> heap{ + worseForHeap}; + for (offset_t i = 0; i + 1 < indptrData.size(); ++i) { + if (indptrData[i + 1] <= indptrData[i]) { + continue; + } + const auto degree = indptrData[i + 1] - indptrData[i]; + degree_entry_t entry{i, degree}; + if (heap.size() < k) { + heap.push(entry); + } else if (better(entry, heap.top())) { + heap.pop(); + heap.push(entry); + } + } + std::vector result; + while (!heap.empty()) { + result.push_back(heap.top()); + heap.pop(); + } + std::sort(result.begin(), result.end(), better); + return result; +} + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/test/CMakeLists.txt b/lance/test/CMakeLists.txt new file mode 100644 index 00000000..2d11983d --- /dev/null +++ b/lance/test/CMakeLists.txt @@ -0,0 +1,5 @@ +if (${BUILD_EXTENSION_TESTS}) + add_lbug_test(lance_utils_test lance_utils_test.cpp) + target_include_directories(lance_utils_test PRIVATE + ${PROJECT_SOURCE_DIR}/extension/lance/src/include) +endif () diff --git a/lance/test/lance_data/indices_follows.lance/_transactions/0-d9417151-fab9-469f-8983-c86ca9730200.txn b/lance/test/lance_data/indices_follows.lance/_transactions/0-d9417151-fab9-469f-8983-c86ca9730200.txn new file mode 100644 index 0000000000000000000000000000000000000000..ffd22ea8b36cc9a41b6e4f587ff95264f74532a6 GIT binary patch literal 186 zcmWeqNwG9BG&eLg)J;oFveY#(vrN;qu(UANO|~#gPP8;PHZU?U*u*x0k;_-elgq*Y z2n@jx!~nAlLBJrz%q-E;#LUzp(a_AoFx4m}#UeS$*x1xiFDEfCIaP>>fl&&G6__-b v4BA;0*oEYT*h&(M(o;(m{=)#H7Hei+iLsFZqXnaoybx<~CQzpYBX$J<=2A8# literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indices_follows.lance/_versions/18446744073709551614.manifest b/lance/test/lance_data/indices_follows.lance/_versions/18446744073709551614.manifest new file mode 100644 index 0000000000000000000000000000000000000000..d0ac44e11deea32422d99b27839a50cb62122c46 GIT binary patch literal 450 zcmdnRz`!7+l45CMXl`g~sGF9UWT|UnW|^jIVQFEkn`~j0oM>roY+z(yu!(H~BbTp` zCzpi*5Ez0Xhyi99f`CDanOUNxiJ7TIqM@0EVX9F|ibZmgv9YP4UQS|Oa;gv$1EUlW zD==v=8ML!1unWlvv6UnirKgrC{D%QXE!NDu5@RC+MhiwEc_G%~OrTB)M(hec0A0^T zzz{A%h6qvDQxc3;JRG;~H(X~EXjtBSnlXYQNy|W&m=M-0smROX=VFBf7Mr1wo`If0 a6%R~+*+|c51tY^gpr0XtiNVLw&lv#tn0~+j literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indices_follows.lance/_versions/latest_version_hint.json b/lance/test/lance_data/indices_follows.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000..491d7344 --- /dev/null +++ b/lance/test/lance_data/indices_follows.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/lance/test/lance_data/indices_follows.lance/data/0000101010000110101111110d66a94658a1681e2dd8cb3351.lance b/lance/test/lance_data/indices_follows.lance/data/0000101010000110101111110d66a94658a1681e2dd8cb3351.lance new file mode 100644 index 0000000000000000000000000000000000000000..3d641689738cabdf9536390841ed84a1af375ad9 GIT binary patch literal 647 zcmZ=@U|{f|05CAf08RM!j}b_MFcT0nL)jo21pWa5E-M0n^0?%P5ocf!0h+)HwdLQx z=j;p&Pk{It5I=>|Ap4n^7#SI{xP!})OHPQbB(W$xwM5}R3@~c3X6BU`8yPTKFmlNY zu@-0MC8tU-Vpkx*&ZQ}&&ZWvFub-0$RHv7kmz-@M_8W;~S0xdKETIe8T$7MsvLTN5ZDLyA2F0iFSAd&-QH515cS5O=SK|X|H*aGEO hL;2t|0A#yD`7rhRpmYHQ{!kj0RG5GX#>dgm8309SouU8$ literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indices_livesin.lance/_transactions/0-f43af6ed-319f-4763-be43-5356d559fb7c.txn b/lance/test/lance_data/indices_livesin.lance/_transactions/0-f43af6ed-319f-4763-be43-5356d559fb7c.txn new file mode 100644 index 0000000000000000000000000000000000000000..99a6809b381320d41a08667e420615e42eb5b571 GIT binary patch literal 150 zcmXxWF%H5Y6oBDEv_nTm2gBfCOfWp&gTe_MJOvamCfdPpGVkO$w8r=^-!FERFjiNx zLI_pS=uo9Xs4QBStqnDe)3d)1XK@-l0O)UvpqKp#U{DiUa+qQ^7UGmW0Rmj7$v^lT$2A3{5O8EKT%s67!N%g%}y6 z7#S3pG?)yYGAXbK$qBKQBo?KomMHv(0Y)v>%)Am~BLhYY#1>L`?@tFk10& z+`8XzolW4z@=0eHBN&pj40MSJ8oiQ=yexh$R){y*42|>*^bD$aU;@lWdPXZ48D;@J M1OZG8K8}9Q03q~bSpWb4 literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indices_livesin.lance/_versions/latest_version_hint.json b/lance/test/lance_data/indices_livesin.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000..491d7344 --- /dev/null +++ b/lance/test/lance_data/indices_livesin.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/lance/test/lance_data/indices_livesin.lance/data/111000001111010011110111230435463bad251acd84149894.lance b/lance/test/lance_data/indices_livesin.lance/data/111000001111010011110111230435463bad251acd84149894.lance new file mode 100644 index 0000000000000000000000000000000000000000..7da59a172eeb206bd4d89a2bbb4b74819862392f GIT binary patch literal 357 zcmZ=@U|{f|05CA{08L;9;(uTOWH2xRF(Xujk%`%z1ui)uwvxo6^wbiC|1iL) z#hRH{Vr*o tjEojc9xMTjOF+H>f=UR*&;X@XpfpTg7UWwX(1p?<3WS+}(8tlw8321MRO0{u literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indptr_follows.lance/_transactions/0-4c1da2c1-d8ee-4b6f-a1dc-6169d17169bd.txn b/lance/test/lance_data/indptr_follows.lance/_transactions/0-4c1da2c1-d8ee-4b6f-a1dc-6169d17169bd.txn new file mode 100644 index 0000000000000000000000000000000000000000..78ab0263053b2b812ac8974a0a58a8c5ec7c6513 GIT binary patch literal 147 zcmWeqF-bN|Ni<3})J?HSP1Q9?GE37H8V7`OffVEl1V9>*s{31h1|I;3=9no zfEWlsB#;T_85mfar6nd>m?Wm8CMH{&o0^&$nV48wS{Rxd>*XZoC8r88GDtBpC@^U- g8Qfq}U=@-UVlF5tQuq%8j9RRjc_qe128nPO-TB$HA$v1M_23%PSy z7#JEF05K4NNFWo;Gcd3;OG`|&FiA{FO-!~lH#IdiGBL5Vv@kR^*2_uEOHLJHWRPNH zP+-zvGPuE{z$zpy#9UBPr0^dG7`0e4^Gb}33>YmKPXV39g-?MHHJu{CXvM>E>wd#^ zHh~lC&YWh9U`Wz3&?P2N^hzr7viP}JAs%EiG}1HBGpOQ$2{0S!8LePsm=5#}1TZo9 IIQlsQ0J$Jz*8l(j literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indptr_follows.lance/_versions/latest_version_hint.json b/lance/test/lance_data/indptr_follows.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000..491d7344 --- /dev/null +++ b/lance/test/lance_data/indptr_follows.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/lance/test/lance_data/indptr_follows.lance/data/01100111110111001001110096faa84adeac97555244998153.lance b/lance/test/lance_data/indptr_follows.lance/data/01100111110111001001110096faa84adeac97555244998153.lance new file mode 100644 index 0000000000000000000000000000000000000000..63da78fa457953fc61b745639aefeaf398640c06 GIT binary patch literal 344 zcmXqDU|{f|05C9!08RM!4@fXD0x=5^vjH(X5dQ-LLPm4Ra!Cs@7nBq!{D%QXE!NDu z5@RC+Mhiv(RxV8;buLvddHtNkyyR59)V$>Ul+3*JVm;^loYLGpS14bIiHkwViHnKB zL5PV(L4sAuOvr@Gh)Wh-n~|YjKw^5TPhw?$X^D^^7rzt>7ZZnq5hKtt57q$2g+OCGBr;#)ipCRFwr$JNKVvEOfxdpO}0o$GPf{GHZe7^+{Bi} literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indptr_livesin.lance/_versions/18446744073709551614.manifest b/lance/test/lance_data/indptr_livesin.lance/_versions/18446744073709551614.manifest new file mode 100644 index 0000000000000000000000000000000000000000..d398377b151276e12a86b3cd0b6b44c01c329d6e GIT binary patch literal 373 zcmbQtz`!7+VwqxOWNMyfs%vIsV4`bckesNSm}X?Gn{1JiWNu-WY+`C+xrr@{%Uj5u z%fi6W&;SI1G?)N!K#Y_`GmFGTlVl5XOM_%{b4yF(Gy@BRR1-5(y`03n1}R1c z1ttw9gBwf=tU}U4%mpPy3jbk%QHwP*uf*8MfYE~S6wp~*_!J0H(gDM`F0JD*v(F#U}=|JB=0270c Hqn|SXEhAzT literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indptr_livesin.lance/_versions/latest_version_hint.json b/lance/test/lance_data/indptr_livesin.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000..491d7344 --- /dev/null +++ b/lance/test/lance_data/indptr_livesin.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/lance/test/lance_data/indptr_livesin.lance/data/011010100111001101011001da68aa4c8790c77993f080e465.lance b/lance/test/lance_data/indptr_livesin.lance/data/011010100111001101011001da68aa4c8790c77993f080e465.lance new file mode 100644 index 0000000000000000000000000000000000000000..2234f6bb7d6a28e816d6b40d0d593852eda12b85 GIT binary patch literal 344 zcmXqDU|{f|05C9!08RM!4@fXD0x=U1GXpUT5dQ-LLPm4Ra!Cs@7nBq!{D%QXE!NDu z5@RC+Mhiv(RxV8;buLvddHtNkyyR59)V$>Ul+3*JVm;^loYLGpS14bIiHkwViHnKB zL5PV(L4sAuOvr@Gh)Wh-n~|YjKw^5TPhw?$X^D^^7rzt>7ZZnq5hKtt57q$2g+O|D&2)0VuAN2*%|CJJv!!R>2!@#;E zQ{_}uiN@1Dkt9q)BvfJ~Wj0fJI)tbHV!;s(fl3G9mxyB+WOS|2*jx&Y8mXYdNQF|^ zl=QCAuC(CI7>`DY`G(_T8G8@`udDd|1KTcY*JlEmVz7d_(-#}EY46|qYwga%fWsPs RuI=uP?(5ctd^-Ezd;kN0MauvH literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/nodes_city.lance/_versions/18446744073709551614.manifest b/lance/test/lance_data/nodes_city.lance/_versions/18446744073709551614.manifest new file mode 100644 index 0000000000000000000000000000000000000000..e20d606d3723b723809fcbf7a131eb10e81e17fa GIT binary patch literal 522 zcmcb|z`!7+l5CljVrrR^qMK-tW~OUmX_TaEX=!Akn`B~XoN8=gY6xU(V%x&VX`WJ4D<}Dcwhp|MtVjo7#SV`V;%yS78 zp+hI@ZLTDFDDxZMym@cFdEY#EHUL!S0KgJe7!KbUz&!_eA{!F^TCz5`{?3M|!)T{oyiFIU!n*j5*DEI5KVLk=auP0KR23gQSI-2C~>Imp%ArmVV;It;`ruB1%l z3EN@x=ZaBdv&dD@?6UzQ!8{2biy{_&6I{o&R6;D`|3%M9ln?lPUB1xFN6;e&w>0|^ i!jXgLIDUxZe#~#i@%a6TF-rKck7&`6)2g1{ls^GyvG8mF literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/nodes_user.lance/_transactions/0-f653a29d-2a71-46c3-8673-e2aae3d3b7e8.txn b/lance/test/lance_data/nodes_user.lance/_transactions/0-f653a29d-2a71-46c3-8673-e2aae3d3b7e8.txn new file mode 100644 index 0000000000000000000000000000000000000000..9fc54d2743b2e6d75286516d1432745271ea1444 GIT binary patch literal 215 zcmYj|O%8%E6ohR-Ojy1ZAu+OWVN7cJ1Erk7g=c6>OGvZ}80=$k9hGS42d3qHC0BZm4TwmTatRVPp#cyANdpiKgdhYY3{8@g4HHe0%}kS$(ku-vlT%I83{%t0jFa?o67!N%g_s!_ znWR930+R-l!CZC)79lAirpy$D|1iL)#hRH{Vr*o}!GtW2(9g@rz`%vad@e%fbKy5%h}wZ6!Dz+9aqE7=bvA(mTUQ@u zj9^I8GSDR^ZuCkj^0N53SRuj8W@w~mpl49U0~262(lc7Y$Z#7N*AT$O;N$4$3;;Tk BnOOh; literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/nodes_user.lance/_versions/latest_version_hint.json b/lance/test/lance_data/nodes_user.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000..491d7344 --- /dev/null +++ b/lance/test/lance_data/nodes_user.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/lance/test/lance_data/nodes_user.lance/data/11010111000101010010000114cc1a4c65bbf919ce5f1ef63b.lance b/lance/test/lance_data/nodes_user.lance/data/11010111000101010010000114cc1a4c65bbf919ce5f1ef63b.lance new file mode 100644 index 0000000000000000000000000000000000000000..84ac02e0186811e13bcf4ad9ac1612507400e1f1 GIT binary patch literal 925 zcmbu7!A`K(!$+CeWK3j|7Z5Fehs{fG@OtmJGk$m zEYu;YjOv8(ST;O4YRPdBMe^xIssX|hw2WzET^6t6v-hX$ebfocSz;zUgDF_x*{alLOyii|0T6MGj?g3`bC^4yEe*6<_s3 z4|}6ZGaSYv^`LXXfQUO_-=5kuR5RV Y`kv0S{_EWQgRgyBmWB*hyU}TW1NN8XR{#J2 literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/rels_follows_flat.lance/_transactions/0-3cdb6ab2-973b-49a6-9deb-16b5d7c90f6a.txn b/lance/test/lance_data/rels_follows_flat.lance/_transactions/0-3cdb6ab2-973b-49a6-9deb-16b5d7c90f6a.txn new file mode 100644 index 0000000000000000000000000000000000000000..50da8087810be7f7e12b515f260efc6b8db11be8 GIT binary patch literal 216 zcmWeqF-}fNGD}P{(zP@Z=e8mAf=Tcjl=CK+2Knp&9X`?xQT5YBbUFB zHK(iSd=^5x5RPn$Bn2q#|RxmQ$0me22FfsTz`Z)stzayGM literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/rels_follows_flat.lance/_versions/latest_version_hint.json b/lance/test/lance_data/rels_follows_flat.lance/_versions/latest_version_hint.json new file mode 100644 index 00000000..491d7344 --- /dev/null +++ b/lance/test/lance_data/rels_follows_flat.lance/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":1} \ No newline at end of file diff --git a/lance/test/lance_data/rels_follows_flat.lance/data/010010001001101111111111503add472da3e238fbab38a584.lance b/lance/test/lance_data/rels_follows_flat.lance/data/010010001001101111111111503add472da3e238fbab38a584.lance new file mode 100644 index 0000000000000000000000000000000000000000..32e3c4adb6e77b8305ca45e36466363f0bfefaec GIT binary patch literal 942 zcmb`F&r1SP5XWcUuB;4E_UK@mC_(h&V75hgh}g2i!y^1wK==rUt>kqp9p3KsEX8C-HOcKe0RPUsX zSQb^w!$rUsWDaqJ8htE}$o>tzN-LQ$RsnHQV2y!dBp6uwk8gcetH&Bl99*mQXWdO4 eJn8&V=R@7^>wJ3uO0VpL_Nq;Vf*<#+R`~>F^6N + +#include "gtest/gtest.h" +#include "storage/lance_utils.h" + +using namespace lbug::lance_extension; + +// ───────────────────────────────────────────────────────────── +// joinPath +// ───────────────────────────────────────────────────────────── +TEST(LanceUtils_JoinPath, EmptyBase) { + EXPECT_EQ("file.lance", LanceUtils::joinPath("", "file.lance")); +} + +TEST(LanceUtils_JoinPath, BaseWithoutTrailingSlash) { + EXPECT_EQ("/base/file.lance", LanceUtils::joinPath("/base", "file.lance")); +} + +TEST(LanceUtils_JoinPath, BaseWithTrailingSlash) { + EXPECT_EQ("/base/file.lance", LanceUtils::joinPath("/base/", "file.lance")); +} + +TEST(LanceUtils_JoinPath, S3URI) { + EXPECT_EQ("s3://bucket/prefix/file.lance", + LanceUtils::joinPath("s3://bucket/prefix", "file.lance")); +} + +// ───────────────────────────────────────────────────────────── +// Layout paths +// ───────────────────────────────────────────────────────────── +TEST(LanceUtils_Paths, NodeTableInDir) { + EXPECT_EQ("/data/nodes_user.lance", LanceUtils::constructNodeTablePath("/data", "user")); +} + +TEST(LanceUtils_Paths, NodeTableSingleDataset) { + EXPECT_EQ("/data/nodes_user.lance", + LanceUtils::constructNodeTablePath("/data/nodes_user.lance", "user")); +} + +TEST(LanceUtils_Paths, CSRPaths) { + auto paths = LanceUtils::constructCSRPaths("/data", "follows"); + EXPECT_EQ("/data/indices_follows.lance", paths.indices); + EXPECT_EQ("/data/indptr_follows.lance", paths.indptr); +} + +TEST(LanceUtils_Paths, FlatRelTableInDir) { + EXPECT_EQ("/data/rels_follows.lance", + LanceUtils::constructFlatRelTablePath("/data", "follows")); +} + +TEST(LanceUtils_Paths, FlatRelTableSingleDataset) { + EXPECT_EQ("/data/rels.lance", + LanceUtils::constructFlatRelTablePath("/data/rels.lance", "follows")); +} + +TEST(LanceUtils_Paths, IsSingleDatasetPath) { + EXPECT_TRUE(LanceUtils::isSingleDatasetPath("/data/nodes_user.lance")); + EXPECT_TRUE(LanceUtils::isSingleDatasetPath("/data/nodes_user.LANCE")); + EXPECT_FALSE(LanceUtils::isSingleDatasetPath("/data")); + EXPECT_FALSE(LanceUtils::isSingleDatasetPath("s3://bucket/prefix")); +} diff --git a/lance/test/test_files/lance.test b/lance/test/test_files/lance.test new file mode 100644 index 00000000..59301aac --- /dev/null +++ b/lance/test/test_files/lance.test @@ -0,0 +1,225 @@ +-DATASET CSV empty +-- + +-CASE LanceScanDataset +-LOAD_DYNAMIC_EXTENSION lance +-LOG ScanNodesViaCall +-STATEMENT CALL LANCE_SCAN('${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data/nodes_user.lance') RETURN * ORDER BY id +---- 4 +75|Noura|25 +100|Adam|30 +250|Karissa|40 +300|Zhang|50 + +-LOG ScanNodesViaLoadFrom +-STATEMENT LOAD FROM '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data/nodes_city.lance' (file_format='lance') RETURN * ORDER BY id +---- 3 +500|Guelph|75000 +600|Kitchener|200000 +700|Waterloo|150000 + +-LOG CopyNodesFromLance +-STATEMENT CREATE NODE TABLE city_copy(id INT32 PRIMARY KEY, name STRING, population INT64) +---- ok +-STATEMENT COPY city_copy FROM '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data/nodes_city.lance' (file_format='lance') +---- ok +-STATEMENT MATCH (c:city_copy) RETURN c.name ORDER BY c.name +---- 3 +Guelph +Kitchener +Waterloo + +-CASE LanceNodeTables +-LOAD_DYNAMIC_EXTENSION lance +-LOG CreateNodeTables +-STATEMENT CREATE NODE TABLE user(id INT32 PRIMARY KEY, name STRING, age INT64) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- ok +-STATEMENT CREATE NODE TABLE city(id INT32 PRIMARY KEY, name STRING, population INT64) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- ok + +-LOG ScanAllUsers +-STATEMENT MATCH (a:user) RETURN a.name ORDER BY a.name +---- 4 +Adam +Karissa +Noura +Zhang + +-LOG FilterByPK +-STATEMENT MATCH (a:user {id: 100}) RETURN a.name, a.age +---- 1 +Adam|30 + +-LOG FilterByProperty +-STATEMENT MATCH (a:user) WHERE a.age > 30 RETURN a.name ORDER BY a.name +---- 2 +Karissa +Zhang + +-LOG CountUsers +-STATEMENT MATCH (a:user) RETURN COUNT(*) +---- 1 +4 + +-CASE LanceRelTables +-LOAD_DYNAMIC_EXTENSION lance +-LOG CreateGraph +-STATEMENT CREATE NODE TABLE user(id INT32 PRIMARY KEY, name STRING, age INT64) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- ok +-STATEMENT CREATE NODE TABLE city(id INT32 PRIMARY KEY, name STRING, population INT64) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- ok +-STATEMENT CREATE REL TABLE follows(FROM user TO user, since INT32) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- ok +-STATEMENT CREATE REL TABLE livesin(FROM user TO city) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- ok + +-LOG OneHopWithProperty +-STATEMENT MATCH (a:user)-[r:follows]->(b:user) RETURN a.name, b.name, r.since ORDER BY a.name, b.name +---- 7 +Adam|Adam|2023 +Adam|Karissa|2020 +Adam|Zhang|2020 +Karissa|Adam|2022 +Karissa|Zhang|2021 +Noura|Adam|2023 +Zhang|Noura|2022 + +-LOG BackwardHop +-STATEMENT MATCH (u)<-[:follows]-(v) WHERE u.id = 300 RETURN v.name ORDER BY v.name +---- 2 +Adam +Karissa + +-LOG UndirectedHop +-STATEMENT MATCH (a:user)-[:livesin]-(c:city) RETURN a.name, c.name ORDER BY a.name +---- 4 +Adam|Waterloo +Karissa|Waterloo +Noura|Guelph +Zhang|Kitchener + +-LOG TwoHopAcrossRels +-STATEMENT MATCH (a:user {id: 100})-[:follows]->(b)-[:livesin]->(c:city) RETURN b.name, c.name ORDER BY b.name +---- 3 +Adam|Waterloo +Karissa|Waterloo +Zhang|Kitchener + +-LOG LivesInEdges +-STATEMENT MATCH (a:user)-[:livesin]->(c:city) RETURN a.name, c.name ORDER BY a.name +---- 4 +Adam|Waterloo +Karissa|Waterloo +Noura|Guelph +Zhang|Kitchener + +-LOG CountAllEdges +-STATEMENT MATCH (a:user)-[r:follows]->(b:user) RETURN COUNT(*) +---- 1 +7 + +-LOG CountNonSelfEdges +-STATEMENT MATCH (a:user)-[:follows]->(b:user) WHERE a.id <> b.id RETURN COUNT(*) +---- 1 +6 + +-LOG VariableLengthOneHop +-STATEMENT MATCH (a:user {id: 100})-[:follows*1..1]->(b:user) RETURN b.name ORDER BY b.name +---- 3 +Adam +Karissa +Zhang + +-LOG VariableLengthTwoHops +-STATEMENT MATCH (a:user {id: 100})-[:follows*1..2]->(b:user) RETURN DISTINCT b.name ORDER BY b.name +---- 4 +Adam +Karissa +Noura +Zhang + +-CASE LanceFlatRelTable +-LOAD_DYNAMIC_EXTENSION lance +-LOG CreateFlatGraph +-STATEMENT CREATE NODE TABLE user(id INT32 PRIMARY KEY, name STRING, age INT64) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- ok +-STATEMENT CREATE REL TABLE fflat(FROM user TO user, since INT32) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data/rels_follows_flat.lance', format = 'lance') +---- ok + +-LOG FlatOneHop +-STATEMENT MATCH (a:user)-[r:fflat]->(b:user) RETURN a.name, b.name, r.since ORDER BY a.name, b.name +---- 7 +Adam|Adam|2023 +Adam|Karissa|2020 +Adam|Zhang|2020 +Karissa|Adam|2022 +Karissa|Zhang|2021 +Noura|Adam|2023 +Zhang|Noura|2022 + +-LOG FlatCount +-STATEMENT MATCH (a:user)-[:fflat]->(b:user) WHERE a.id <> b.id RETURN COUNT(*) +---- 1 +6 + +-CASE LanceMixedTables +-LOAD_DYNAMIC_EXTENSION lance +-LOG CreateLanceNodeTable +-STATEMENT CREATE NODE TABLE user(id INT32 PRIMARY KEY, name STRING, age INT64) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- ok +-LOG CreateRegularNodeTable +-STATEMENT CREATE NODE TABLE person(id INT64 PRIMARY KEY, name STRING) +---- ok + +-LOG LanceRelWithRegularNodeFails +-STATEMENT CREATE REL TABLE rel(FROM user TO person) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- error +Binder exception: Cannot mix lance tables with non-lance tables in CREATE REL TABLE. + +-LOG RegularRelWithLanceNodeFails +-STATEMENT CREATE REL TABLE rel2(FROM user TO user) +---- error +Binder exception: Cannot mix lance tables with non-lance tables in CREATE REL TABLE. + +-CASE LanceInvalidStorage +-LOAD_DYNAMIC_EXTENSION lance +-LOG MissingDatasetDirFails +-STATEMENT CREATE NODE TABLE t(id INT64 PRIMARY KEY) WITH (storage = '/nonexistent/dir/that/does/not/exist', format = 'lance') +---- error(regex) +.*[Nn]o such file.*|.*[Cc]annot open.*|.*[Nn]ot found.*|.*nodes_t\.lance.* + +-LOG NotALanceDatasetFails +-STATEMENT CREATE NODE TABLE t2(id INT64 PRIMARY KEY) WITH (storage = '${LBUG_ROOT_DIRECTORY}/src', format = 'lance') +---- error(regex) +.*[Nn]ot a Lance dataset.*|.*[Cc]annot open.*|.*nodes_t2\.lance.* + +-LOG MissingPKColumnFails +-STATEMENT CREATE NODE TABLE t3(zzz INT32 PRIMARY KEY, name STRING) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data/nodes_user.lance', format = 'lance') +---- error +Runtime exception: Lance dataset '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data/nodes_user.lance' has no primary key column 'zzz' for node table 't3' + +-LOG UnknownFormatFails +-STATEMENT CREATE NODE TABLE t4(id INT64 PRIMARY KEY) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'foo') +---- error +Binder exception: Unsupported storage format 'foo'. Valid options are: icebug-disk, lance. + +-CASE LanceRemoteURIDeferred +-LOAD_DYNAMIC_EXTENSION lance +-LOG RemoteCreateSucceeds +-STATEMENT CREATE NODE TABLE t(id INT64 PRIMARY KEY) WITH (storage = 's3://my-bucket/data', format = 'lance') +---- ok +-LOG RemoteScanFails +-STATEMENT MATCH (n:t) RETURN * +---- error(regex) +.*Failed to open Lance dataset.* + +-CASE LanceReadOnly +-LOAD_DYNAMIC_EXTENSION lance +-LOG CreateLanceNodeTable +-STATEMENT CREATE NODE TABLE user(id INT32 PRIMARY KEY, name STRING, age INT64) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'lance') +---- ok + +-LOG InsertIntoLanceTableFails +-STATEMENT CREATE (a:user {id: 1, name: 'X', age: 1}) +---- error +Runtime exception: Cannot insert into lance-backed node table From 7856949a37bdc467a2da6848cbdcb6dff9130639 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Mon, 26 Oct 2026 22:49:00 -0700 Subject: [PATCH 2/6] feat(lance): read Lance natively via Rust FFI instead of DuckDB Replace the DuckDB-mediated reads with a native reader: new rust/ crate (lance_ladybug_ffi, Lance 13 + Arrow 58, same versions as lance-duckdb) exposing open/count/schema/scan-to-Arrow-batches over a C ABI, built and linked by CMake following the lance-duckdb Cargo integration. Batches cross the FFI via the Arrow C data interface and decode with core ArrowConverter. - Tables and LANCE_SCAN keep their structure; only the read primitive swaps (SQL CASTs become exact-type validation at CREATE, which now also enforces the stamped lance_version marker). - Fixtures regenerated with rust/examples/gen_fixtures.rs (identical rows, native int widths, v1 markers) plus a v999 dataset for the version-mismatch test; e2e expectations unchanged. - Drop the DuckDB connector, loader, installer and find_package. --- lance/CMakeLists.txt | 179 +- lance/README.md | 42 +- lance/rust/.gitignore | 1 + lance/rust/Cargo.lock | 7443 +++++++++++++++++ lance/rust/Cargo.toml | 28 + lance/rust/examples/gen_fixtures.rs | 152 + lance/rust/src/error.rs | 70 + lance/rust/src/ffi.rs | 578 ++ lance/rust/src/lib.rs | 5 + lance/rust/src/runtime.rs | 22 + lance/scripts/gen_test_data.sh | 71 +- lance/src/connector/CMakeLists.txt | 9 - lance/src/connector/lance_connector.cpp | 83 - lance/src/function/CMakeLists.txt | 5 +- lance/src/function/lance_scan.cpp | 86 +- lance/src/include/connector/lance_connector.h | 35 - lance/src/include/function/lance_scan.h | 52 +- lance/src/include/storage/lance_constants.h | 7 +- lance/src/include/storage/lance_ffi.h | 128 + lance/src/include/storage/lance_node_table.h | 9 +- lance/src/include/storage/lance_reader.h | 94 +- lance/src/include/storage/lance_rel_table.h | 7 +- lance/src/installer/CMakeLists.txt | 11 - lance/src/installer/lance_install_func.cpp | 16 - lance/src/main/CMakeLists.txt | 4 +- lance/src/main/lance_extension.cpp | 2 - lance/src/storage/lance_node_table.cpp | 91 +- lance/src/storage/lance_reader.cpp | 248 +- lance/src/storage/lance_rel_table.cpp | 125 +- ...0-555c15fd-2050-448e-a9e0-16be92c04577.txn | Bin 0 -> 182 bytes ...0-d9417151-fab9-469f-8983-c86ca9730200.txn | Bin 186 -> 0 bytes .../_versions/18446744073709551614.manifest | Bin 450 -> 463 bytes ...10101111110d66a94658a1681e2dd8cb3351.lance | Bin 647 -> 0 bytes ...0000011100a2f75046f78580c71e07d60c55.lance | Bin 0 -> 738 bytes ...0-901c37e7-0288-4445-8bdf-95ade946db17.txn | Bin 0 -> 148 bytes ...0-f43af6ed-319f-4763-be43-5356d559fb7c.txn | Bin 150 -> 0 bytes .../_versions/18446744073709551614.manifest | Bin 379 -> 396 bytes ...0101111000577a304c0cbe4340451ec9bc0b.lance | Bin 0 -> 376 bytes ...0011110111230435463bad251acd84149894.lance | Bin 357 -> 0 bytes ...0-4c1da2c1-d8ee-4b6f-a1dc-6169d17169bd.txn | Bin 147 -> 0 bytes ...0-b6cd471b-30be-43e0-a05c-b13623fe1848.txn | Bin 0 -> 145 bytes .../_versions/18446744073709551614.manifest | Bin 373 -> 390 bytes ...001001110096faa84adeac97555244998153.lance | Bin 344 -> 0 bytes ...100010001120d94b4d57a28e28068b023bc1.lance | Bin 0 -> 363 bytes ...0-36465627-3a3e-48f9-8397-cae74b97dbf6.txn | Bin 0 -> 145 bytes ...0-9d2257f5-6204-40ca-af23-c8db786c4549.txn | Bin 147 -> 0 bytes .../_versions/18446744073709551614.manifest | Bin 373 -> 390 bytes ...0010101001eddf4d4e92b6dbff387abd017b.lance | Bin 0 -> 363 bytes ...1101011001da68aa4c8790c77993f080e465.lance | Bin 344 -> 0 bytes ...0-c9bd59dd-a0f6-492b-9928-b493e3851b49.txn | Bin 222 -> 0 bytes ...0-e868a4a2-b0e3-409f-8d58-5f152cb5b5ec.txn | Bin 0 -> 216 bytes .../_versions/18446744073709551614.manifest | Bin 522 -> 531 bytes ...11111010066a1df433fb41c8a6e5f945ddc.lance} | Bin 932 -> 947 bytes ...0-8a8b34fe-e4c9-4536-92ef-7991b093f482.txn | Bin 0 -> 209 bytes ...0-f653a29d-2a71-46c3-8673-e2aae3d3b7e8.txn | Bin 215 -> 0 bytes .../_versions/18446744073709551614.manifest | Bin 508 -> 517 bytes ...001101011c169584412b3ee42b068362104.lance} | Bin 925 -> 940 bytes ...0-2e032a0f-7af1-4c64-8f6c-2e5e7fba6eeb.txn | Bin 0 -> 144 bytes .../_versions/18446744073709551614.manifest | Bin 0 -> 390 bytes .../_versions/latest_version_hint.json | 1 + ...11001010013e62334444b3a58d8f7ebf047b.lance | Bin 0 -> 222 bytes ...0-3cdb6ab2-973b-49a6-9deb-16b5d7c90f6a.txn | Bin 216 -> 0 bytes ...0-d9206ba4-d346-4ecc-8c65-432516090d97.txn | Bin 0 -> 210 bytes .../_versions/18446744073709551614.manifest | Bin 510 -> 519 bytes ...011011010042350c446c888cc9e91705b747.lance | Bin 0 -> 1031 bytes ...1111111111503add472da3e238fbab38a584.lance | Bin 942 -> 0 bytes lance/test/test_files/lance.test | 5 + 67 files changed, 8987 insertions(+), 622 deletions(-) create mode 100644 lance/rust/.gitignore create mode 100644 lance/rust/Cargo.lock create mode 100644 lance/rust/Cargo.toml create mode 100644 lance/rust/examples/gen_fixtures.rs create mode 100644 lance/rust/src/error.rs create mode 100644 lance/rust/src/ffi.rs create mode 100644 lance/rust/src/lib.rs create mode 100644 lance/rust/src/runtime.rs delete mode 100644 lance/src/connector/CMakeLists.txt delete mode 100644 lance/src/connector/lance_connector.cpp delete mode 100644 lance/src/include/connector/lance_connector.h create mode 100644 lance/src/include/storage/lance_ffi.h delete mode 100644 lance/src/installer/CMakeLists.txt delete mode 100644 lance/src/installer/lance_install_func.cpp create mode 100644 lance/test/lance_data/indices_follows.lance/_transactions/0-555c15fd-2050-448e-a9e0-16be92c04577.txn delete mode 100644 lance/test/lance_data/indices_follows.lance/_transactions/0-d9417151-fab9-469f-8983-c86ca9730200.txn delete mode 100644 lance/test/lance_data/indices_follows.lance/data/0000101010000110101111110d66a94658a1681e2dd8cb3351.lance create mode 100644 lance/test/lance_data/indices_follows.lance/data/011100111101010000011100a2f75046f78580c71e07d60c55.lance create mode 100644 lance/test/lance_data/indices_livesin.lance/_transactions/0-901c37e7-0288-4445-8bdf-95ade946db17.txn delete mode 100644 lance/test/lance_data/indices_livesin.lance/_transactions/0-f43af6ed-319f-4763-be43-5356d559fb7c.txn create mode 100644 lance/test/lance_data/indices_livesin.lance/data/101101000001010101111000577a304c0cbe4340451ec9bc0b.lance delete mode 100644 lance/test/lance_data/indices_livesin.lance/data/111000001111010011110111230435463bad251acd84149894.lance delete mode 100644 lance/test/lance_data/indptr_follows.lance/_transactions/0-4c1da2c1-d8ee-4b6f-a1dc-6169d17169bd.txn create mode 100644 lance/test/lance_data/indptr_follows.lance/_transactions/0-b6cd471b-30be-43e0-a05c-b13623fe1848.txn delete mode 100644 lance/test/lance_data/indptr_follows.lance/data/01100111110111001001110096faa84adeac97555244998153.lance create mode 100644 lance/test/lance_data/indptr_follows.lance/data/10001100111001100010001120d94b4d57a28e28068b023bc1.lance create mode 100644 lance/test/lance_data/indptr_livesin.lance/_transactions/0-36465627-3a3e-48f9-8397-cae74b97dbf6.txn delete mode 100644 lance/test/lance_data/indptr_livesin.lance/_transactions/0-9d2257f5-6204-40ca-af23-c8db786c4549.txn create mode 100644 lance/test/lance_data/indptr_livesin.lance/data/000010101010100010101001eddf4d4e92b6dbff387abd017b.lance delete mode 100644 lance/test/lance_data/indptr_livesin.lance/data/011010100111001101011001da68aa4c8790c77993f080e465.lance delete mode 100644 lance/test/lance_data/nodes_city.lance/_transactions/0-c9bd59dd-a0f6-492b-9928-b493e3851b49.txn create mode 100644 lance/test/lance_data/nodes_city.lance/_transactions/0-e868a4a2-b0e3-409f-8d58-5f152cb5b5ec.txn rename lance/test/lance_data/nodes_city.lance/data/{111101101100001100010011a80be542a0a09797b48042e19a.lance => 10001100110100011111010066a1df433fb41c8a6e5f945ddc.lance} (60%) create mode 100644 lance/test/lance_data/nodes_user.lance/_transactions/0-8a8b34fe-e4c9-4536-92ef-7991b093f482.txn delete mode 100644 lance/test/lance_data/nodes_user.lance/_transactions/0-f653a29d-2a71-46c3-8673-e2aae3d3b7e8.txn rename lance/test/lance_data/nodes_user.lance/data/{11010111000101010010000114cc1a4c65bbf919ce5f1ef63b.lance => 010010000101100001101011c169584412b3ee42b068362104.lance} (60%) create mode 100644 lance/test/lance_data/nodes_wrongversion.lance/_transactions/0-2e032a0f-7af1-4c64-8f6c-2e5e7fba6eeb.txn create mode 100644 lance/test/lance_data/nodes_wrongversion.lance/_versions/18446744073709551614.manifest create mode 100644 lance/test/lance_data/nodes_wrongversion.lance/_versions/latest_version_hint.json create mode 100644 lance/test/lance_data/nodes_wrongversion.lance/data/0001000111111011001010013e62334444b3a58d8f7ebf047b.lance delete mode 100644 lance/test/lance_data/rels_follows_flat.lance/_transactions/0-3cdb6ab2-973b-49a6-9deb-16b5d7c90f6a.txn create mode 100644 lance/test/lance_data/rels_follows_flat.lance/_transactions/0-d9206ba4-d346-4ecc-8c65-432516090d97.txn create mode 100644 lance/test/lance_data/rels_follows_flat.lance/data/00011110001101011011010042350c446c888cc9e91705b747.lance delete mode 100644 lance/test/lance_data/rels_follows_flat.lance/data/010010001001101111111111503add472da3e238fbab38a584.lance diff --git a/lance/CMakeLists.txt b/lance/CMakeLists.txt index 104688ed..ad3b2500 100644 --- a/lance/CMakeLists.txt +++ b/lance/CMakeLists.txt @@ -1,23 +1,165 @@ -if (WIN32 OR BUILD_STATIC_EXTENSION) - set(DuckDB_USE_STATIC_LIBS ON) -else () - set(DuckDB_USE_STATIC_LIBS OFF) -endif () -find_package(DuckDB REQUIRED) +cmake_minimum_required(VERSION 3.14...3.29) + +# Set extension name here +set(TARGET_NAME lance) + +set(EXTENSION_NAME ${TARGET_NAME}_extension) + +project(${TARGET_NAME}) +include_directories(src/include) + +# Build and link the Rust staticlib (lance_ladybug_ffi), following the +# lance-duckdb CMake/Cargo integration: native Lance reads without DuckDB. +set(RUST_PLATFORM_TARGET "") + +set(LANCE_PLATFORM "$ENV{LANCE_PLATFORM}") +if(LANCE_PLATFORM STREQUAL "linux_amd64_musl") + set(RUST_PLATFORM_TARGET "x86_64-unknown-linux-musl") +elseif(LANCE_PLATFORM STREQUAL "linux_arm64_musl") + set(RUST_PLATFORM_TARGET "aarch64-unknown-linux-musl") +elseif(LANCE_PLATFORM STREQUAL "windows_amd64_mingw") + set(RUST_PLATFORM_TARGET "x86_64-pc-windows-gnu") +endif() + +if(RUST_PLATFORM_TARGET STREQUAL "") + if("${OS_NAME}" STREQUAL "linux") + if("${OS_ARCH}" STREQUAL "arm64") + set(RUST_PLATFORM_TARGET "aarch64-unknown-linux-gnu") + else() + set(RUST_PLATFORM_TARGET "x86_64-unknown-linux-gnu") + endif() + elseif("${OS_NAME}" STREQUAL "osx") + if("${OSX_BUILD_ARCH}" STREQUAL "arm64") + set(RUST_PLATFORM_TARGET "aarch64-apple-darwin") + elseif("${OSX_BUILD_ARCH}" STREQUAL "x86_64") + set(RUST_PLATFORM_TARGET "x86_64-apple-darwin") + elseif("${OS_ARCH}" STREQUAL "arm64") + set(RUST_PLATFORM_TARGET "aarch64-apple-darwin") + else() + set(RUST_PLATFORM_TARGET "x86_64-apple-darwin") + endif() + elseif("${OS_NAME}" STREQUAL "windows") + if("${OS_ARCH}" STREQUAL "arm64") + set(RUST_PLATFORM_TARGET "aarch64-pc-windows-msvc") + else() + set(RUST_PLATFORM_TARGET "x86_64-pc-windows-msvc") + endif() + endif() +endif() + +if(RUST_PLATFORM_TARGET STREQUAL "") + if(APPLE) + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64|aarch64") + set(RUST_PLATFORM_TARGET "aarch64-apple-darwin") + else() + set(RUST_PLATFORM_TARGET "x86_64-apple-darwin") + endif() + elseif(UNIX) + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64|aarch64") + set(RUST_PLATFORM_TARGET "aarch64-unknown-linux-gnu") + else() + set(RUST_PLATFORM_TARGET "x86_64-unknown-linux-gnu") + endif() + elseif(WIN32) + set(RUST_PLATFORM_TARGET "x86_64-pc-windows-msvc") + endif() +endif() + +if(WIN32) + if(RUST_PLATFORM_TARGET MATCHES "windows-gnu") + set(RUST_STATICLIB_NAME "liblance_ladybug_ffi.a") + else() + set(RUST_STATICLIB_NAME "lance_ladybug_ffi.lib") + endif() +else() + set(RUST_STATICLIB_NAME "liblance_ladybug_ffi.a") +endif() + +set(RUST_TARGET_DIR "${CMAKE_BINARY_DIR}/cargo") +set(RUST_DEBUG_LIB + "${RUST_TARGET_DIR}/${RUST_PLATFORM_TARGET}/debug/${RUST_STATICLIB_NAME}") +set(RUST_RELEASE_LIB + "${RUST_TARGET_DIR}/${RUST_PLATFORM_TARGET}/release/${RUST_STATICLIB_NAME}") + +# Cargo discovers Rust modules; CMake must notice changes to every module before +# deciding whether to invoke Cargo, including newly added FFI modules. +file(GLOB_RECURSE RUST_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/rust/src/*.rs") +set(RUST_FFI_DEPENDS ${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml + ${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.lock ${RUST_SOURCES}) + +set(RUST_CARGO_ENV CARGO_TARGET_DIR=${RUST_TARGET_DIR} + MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}) +if(NOT "$ENV{PROTOC}" STREQUAL "") + list(APPEND RUST_CARGO_ENV "PROTOC=$ENV{PROTOC}") +endif() + +# Platform libs required by Rust deps (e.g., TLS, timezone). +if(APPLE) + if(NOT DEFINED CMAKE_OSX_DEPLOYMENT_TARGET OR CMAKE_OSX_DEPLOYMENT_TARGET + STREQUAL "") + set(CMAKE_OSX_DEPLOYMENT_TARGET + 13.3 + CACHE STRING "Minimum OS X deployment version" FORCE) + endif() + set(PLATFORM_LIBS "-framework CoreFoundation" "-framework IOKit" + "-framework SystemConfiguration" "-framework Security") +elseif(UNIX) + set(PLATFORM_LIBS m c) +elseif(WIN32) + set(PLATFORM_LIBS + ws2_32 + userenv + advapi32 + crypt32 + ntdll + bcrypt + ncrypt + secur32) +endif() + +# Building both debug and release Rust artifacts is expensive and can trigger +# OOMs in CI. For single-config generators (Ninja/Makefiles), only build the +# active configuration's artifact. +if(CMAKE_CONFIGURATION_TYPES) + add_custom_command( + OUTPUT "${RUST_DEBUG_LIB}" "${RUST_RELEASE_LIB}" + COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} cargo build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --target=${RUST_PLATFORM_TARGET} + COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} cargo build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --release --target=${RUST_PLATFORM_TARGET} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + DEPENDS ${RUST_FFI_DEPENDS} + VERBATIM) + add_custom_target(lance_ladybug_ffi_build DEPENDS "${RUST_DEBUG_LIB}" "${RUST_RELEASE_LIB}") +else() + if(NOT DEFINED CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "") + set(CMAKE_BUILD_TYPE Release) + endif() + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + add_custom_command( + OUTPUT "${RUST_DEBUG_LIB}" + COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} cargo build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --target=${RUST_PLATFORM_TARGET} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + DEPENDS ${RUST_FFI_DEPENDS} + VERBATIM) + add_custom_target(lance_ladybug_ffi_build DEPENDS "${RUST_DEBUG_LIB}") + else() + add_custom_command( + OUTPUT "${RUST_RELEASE_LIB}" + COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} cargo build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --release --target=${RUST_PLATFORM_TARGET} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + DEPENDS ${RUST_FFI_DEPENDS} + VERBATIM) + add_custom_target(lance_ladybug_ffi_build DEPENDS "${RUST_RELEASE_LIB}") + endif() +endif() include_directories( ${PROJECT_SOURCE_DIR}/src/include ${CMAKE_BINARY_DIR}/src/include src/include - ${PROJECT_SOURCE_DIR}/extension/duckdb/src/include - ${PROJECT_SOURCE_DIR}/extension/httpfs/src/include - ${DuckDB_INCLUDE_DIRS} ) -add_subdirectory(src/connector) add_subdirectory(src/function) add_subdirectory(src/storage) -add_subdirectory(src/installer) add_subdirectory(src/main) add_subdirectory(test) @@ -25,15 +167,6 @@ build_extension_lib(${BUILD_STATIC_EXTENSION} "lance") target_link_libraries(lbug_${EXTENSION_LIB_NAME}_extension PRIVATE - ${DuckDB_LIBRARIES}) - -if (NOT WIN32 AND NOT BUILD_STATIC_EXTENSION) - add_library(lance_loader - SHARED - ${PROJECT_SOURCE_DIR}/extension/duckdb/src/loader/duckdb_loader.cpp) - set_extension_properties(lance_loader lance_loader lance) -endif () - -if (APPLE AND NOT BUILD_STATIC_EXTENSION) - set_apple_dynamic_lookup(lance_loader) -endif () + debug "${RUST_DEBUG_LIB}" optimized "${RUST_RELEASE_LIB}" + ${PLATFORM_LIBS}) +add_dependencies(lbug_${EXTENSION_LIB_NAME}_extension lance_ladybug_ffi_build) diff --git a/lance/README.md b/lance/README.md index fd94d571..27c29265 100644 --- a/lance/README.md +++ b/lance/README.md @@ -47,15 +47,15 @@ COPY person FROM 'graph/nodes_user.lance' (file_format = 'lance'); ## How it works -**Lance I/O goes through DuckDB.** The extension embeds an in-memory DuckDB -that loads the `lance` core extension — i.e. the exact engine built by -[lance-duckdb](https://github.com/lance-duckdb) (this extension tracks what -DuckDB `v2.0-cyanoptera` ships, the same line lance-duckdb `main` targets). -Reads are range queries (`SELECT FROM '.lance' LIMIT/OFFSET`) -which lance-duckdb pushes into the Lance scan, so morsel and CSR range scans -stay efficient. This mirrors how the `delta`/`iceberg` extensions reuse -DuckDB's Delta/Iceberg extensions, and it means no Rust toolchain or Lance -linkage is required to build or run this extension. +**Lance I/O is native.** The extension links the Lance Rust crates directly +(same versions as [lance-duckdb](https://github.com/lance-duckdb) `main`: +Lance 13, Arrow 58) through a small C FFI (`rust/`: open/count/schema/scan +to Arrow batches), with no DuckDB layer in between. Reads are dataset scans +with projection + limit/offset pushed into Lance, so morsel and CSR range +scans stay efficient; batches cross the FFI via the Arrow C data interface +and decode with core's `ArrowConverter` (the same machinery backing +`arrow://` tables). Building needs a Rust toolchain, protoc, and crates.io +access (see `rust/`); the CMake build drives cargo automatically. **Storage layout mirrors icebug-disk** (`IceDiskNodeTable`/`IceDiskRelTable` are the templates for `LanceNodeTable`/`LanceRelTable`): positional node @@ -70,7 +70,7 @@ properties read as NULL). **Plugging into core.** Storage table types cannot live behind a plain table function (graph traversal needs CSR extend, degree summaries, PK lookup), so core provides a small extension hook surface (see -`src/include/storage/table/external_table_hooks.h`): +`src/include/storage/table/external_table_hooks.h`, LadybugDB/ladybug#1445): - `StorageFormat::LANCE` + `ExternalTableHookRegistry` (factories + scan hooks, keyed by format and by table); @@ -92,9 +92,9 @@ edge tables at query time. This extension takes the complementary, storage-first approach: the CSR (`indices` + `indptr`) is **materialized as Lance datasets on disk** (the icebug-disk layout translated to Lance), so graphs are portable files that any Lance reader can open, while traversal -still resolves hops through CSR range lookups instead of joins. A native -(Rust, zero-DuckDB) reader reusing lance-duckdb's `rust/` crates is the -natural next step; the table layer is already decoupled behind `LanceReader`. +still resolves hops through CSR range lookups instead of joins. Reads go +through the extension's own native reader (`rust/` + `LanceReader`); a +Rust-free build is not a goal. ## Regenerating test fixtures @@ -102,16 +102,20 @@ natural next step; the table layer is already decoupled behind `LanceReader`. ./extension/lance/scripts/gen_test_data.sh ``` -regenerates `extension/lance/test/lance_data/` with the DuckDB CLI. Content -mirrors `dataset/ice-disk-test` row-for-row. +regenerates `extension/lance/test/lance_data/` with the extension's own Rust +generator (`rust/examples/gen_fixtures.rs`, same Lance version as the reader). +Content mirrors `dataset/ice-disk-test` row-for-row; every dataset is stamped +with `lance_version = v1` schema metadata (plus a `v999` dataset for the +version-mismatch error test). ## Limitations (v1) - Read-only tables; no `COPY ... TO ... (FORMAT lance)` write path yet (write - Lance datasets with DuckDB/pylance, or vote for the native writer). -- `STRUCT`/`MAP`/`UNION` property columns are rejected at `CREATE` with a - clear error; scalars, `LIST`/`ARRAY` (incl. fixed-size vectors), temporal, - `UUID`, `DECIMAL`, and `BLOB` are supported. + Lance datasets with pylance/DuckDB, or vote for the native writer). +- Declared property types must match the dataset exactly (no silent coercion; + endpoint offset columns must be `INT64`). Whatever core's Arrow converter + rejects (float16, time32/64, maps, ...) fails fast at `CREATE`; timezone-aware + timestamps and durations are rejected in the FFI for the same reason. - Object-store datasets skip validation at `CREATE` (credentials may be configured later) and fail at first scan, mirroring icebug-disk. - Semi-mask morsel skipping is not implemented (masks still filter); GDS diff --git a/lance/rust/.gitignore b/lance/rust/.gitignore new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/lance/rust/.gitignore @@ -0,0 +1 @@ +/target diff --git a/lance/rust/Cargo.lock b/lance/rust/Cargo.lock new file mode 100644 index 00000000..b47defa6 --- /dev/null +++ b/lance/rust/Cargo.lock @@ -0,0 +1,7443 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arc-swap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "arrow" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c943b9bec582e47f8394e44713c3cbfbe645fd049e3a7771585f54849c710fe6" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b13b85a6a547a1669114e418f6ab3c3e39c2edb1b8be5ab2889e5739a433ae9" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b11eb61b95868592ad6edbd537465a6940eb3c81a875addecec24283301b267" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53ee609a07eb721c2e4a7669d7bb88e2106c560e7dec1219096fecf5eae356d1" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9575323ffeb4bc6955f39bf8716f734671a32780f1fac9ba6c2891048fc0ea1f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50659e76e264937e967167e08d76b1d50b1c162f723c5a3c0527d28c4cdf0d56" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10791765e6927cdd8ace5bf0e51a69f38a1c3aded2db93561b003c332d02ee2a" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165051f1f996f19b50c4d2cdb7ed02ddd400d8d984b422bf6848507eee099c3d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0efe21cd317f22e6b8d7eecc4ed71a043e7269a7bc313a8827c7fb8733866ca" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap 2.14.2", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908a1b5aeefce5c6eb2a0a4e98f4ccacbd441fe0ed23fb799e634cd0135821e5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5383b1676492a509032f06fa4ca9522c886036aec4ebe8261bebfbce904af667" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8506ad465ebb9483bb6efa1e2e78a313549122c6dd67dc2f7087297b762eaede" +dependencies = [ + "bitflags 2.13.2", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0f0a0d641356ae2df829af5d635ed9fb93fbeb0d39f3239487cb5fdb37a6ef" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd245a358d52e77bfcb99dc3e21f04b947383271b7a0b9ca8feec43e9039234d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-compression" +version = "0.4.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-recursion" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8abc12baad266b1c8cec146854c195b5864b4221d4b2ca7296a7ae82d9e451" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "async_cell" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447ab28afbb345f5408b120702a44e5529ebf90b1796ec76e9528df8e288e6c2" +dependencies = [ + "loom", +] + +[[package]] +name = "asyncband" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2050892d4691cf110586070786712b504985a4d37ed64a9cd8a402718e1706" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-config" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "939e256cfe0ed68e96d32f194339d88210d5c10685cff0787400158f852c6819" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http", + "sha1 0.10.7", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576e8da35751dc85409ec1cd15999f692c277a1fbe9336780fe6f15e49dd637d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "aws-runtime" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a0cc2144962cc5e358467b16e3d82981b1d424a7c3d34602be0036103814a1a" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.117.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5316813498fa9c403877007934ae448aa44b18136f58051f8fa3c5b79b44bdc6" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.119.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31db35e5662787b786daba15f431e4393c7ea512b70fa112bd048d1f3ecf877a" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.122.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a10491638d9e560db90c484dbc77d73b8ad202780cdc9b9e92180603acb07ada" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac 0.13.0", + "http", + "percent-encoding", + "sha2 0.11.1", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165f6265342a313ad6f7a1ac17a0cdba30c522039d3f41d8d6871ce8a2b34ecd" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http", +] + +[[package]] +name = "aws-smithy-types" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "http", + "http-body", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "ryu", + "serde", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0d970d3947bb064d2ae1d823efa186c1dbde88f88afa8f6549946e5d8deee" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "compression-codecs" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "const-str" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" + +[[package]] +name = "const_panic" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9603f79528ece8163c496f8932121cb36cfe46259e9c907bb3d8205139d7caa3" +dependencies = [ + "typewit", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "countio" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" +dependencies = [ + "futures-io", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.1", +] + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + +[[package]] +name = "ctutils" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "futures", + "indexmap 2.14.2", + "itertools 0.14.0", + "log", + "object_store 0.13.2", + "parking_lot", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion-catalog" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store 0.13.2", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools 0.14.0", + "log", + "object_store 0.13.2", +] + +[[package]] +name = "datafusion-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.14.0", + "libc", + "log", + "object_store 0.13.2", + "sqlparser", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store 0.13.2", + "parking_lot", + "rand 0.9.5", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "object_store 0.13.2", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store 0.13.2", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store 0.13.2", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-doc" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" + +[[package]] +name = "datafusion-execution" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store 0.13.2", + "parking_lot", + "rand 0.9.5", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap 2.14.2", + "itertools 0.14.0", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap 2.14.2", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-functions" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" +dependencies = [ + "arrow", + "arrow-buffer", + "base64 0.22.1", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hex", + "itertools 0.14.0", + "log", + "md-5", + "memchr", + "num-traits", + "rand 0.9.5", + "regex", + "sha2 0.11.1", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "foldhash 0.2.0", + "half", + "log", + "num-traits", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hashbrown 0.17.1", + "itertools 0.14.0", + "itoa", + "log", + "memchr", +] + +[[package]] +name = "datafusion-functions-table" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-functions-window" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" +dependencies = [ + "datafusion-doc", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "datafusion-optimizer" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap 2.14.2", + "itertools 0.14.0", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.14.0", + "parking_lot", + "petgraph", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.14.0", + "parking_lot", + "pin-project", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-plan" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.14.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-pruning" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "log", +] + +[[package]] +name = "datafusion-session" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" +dependencies = [ + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-sql" +version = "54.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", + "indexmap 2.14.2", + "log", + "regex", + "sqlparser", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.2", + "objc2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "ethnum" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fast-float2" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e8948ce679d00a02a94739ea185595dca7118ed04feb991127e443bd3d761f" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.13.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "frostem" +version = "1.20260821.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2488a10218a17366ce6bd38b6d081b4387a9548d350029db9feddc9bb44d6d" + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "fsst" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de52c5e8a41983a44223740e4322122b5a9111909215ebdb4abd9b3b4cf04a67" +dependencies = [ + "arrow-array", + "rand 0.9.5", +] + +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" +dependencies = [ + "utf8-ranges", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gearhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616e8f476a586f1b078d9eece21f7a071f27997e709e2b471f5697deea53bda9" + +[[package]] +name = "generator" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be81b31c9ce4caf7f3178eeb90662bb1864d3ad363ec6bdada3201adff19c37" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "bytemuck", + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapify" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hf-xet" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24754aef4f6c3d57bc4a0dc2648ca74487050eda01c361b3eefa11feeafd662" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "http", + "more-asserts", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tokio_with_wasm", + "tracing", + "uuid", + "xet-client", + "xet-core-structures", + "xet-data", + "xet-runtime", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "hyperloglogplus" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "621debdf94dcac33e50475fdd76d34d5ea9c0362a834b9db08c3024696c1fbe3" +dependencies = [ + "serde", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "serde", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "406e9ae5b95d8aa37db6293405931b766a6172d1848daee22d70eb44aa78a43c" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3632ebbd0c5c3637ba1a40bdf8288f2cb9d2268e73441a69d5fc73d7cfbaa87c" + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "serde", + "stable_deref_trait", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_segmenter" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b195dc605f81aa181bec9b3bd14e26cc71b903e4644f6af578ca9772c7a710" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913a7d294ad056474749152107309fdee3ec17d2482d1ab70fdc61a81c64b9a3" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "io-uring" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "jsonb" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953" +dependencies = [ + "byteorder", + "ethnum", + "fast-float2", + "itoa", + "jiff", + "nom", + "num-traits", + "ordered-float", + "rand 0.9.5", + "serde", + "serde_json", + "zmij", +] + +[[package]] +name = "konst" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" +dependencies = [ + "const_panic", + "konst_proc_macros", + "typewit", +] + +[[package]] +name = "konst_proc_macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" + +[[package]] +name = "lance" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e8dc0a0759fa0fcdbb970eec48ade5d02dbbd255147ff1f7b06d402d2f86b97" +dependencies = [ + "arc-swap", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "async_cell", + "blake3", + "byteorder", + "bytes", + "chrono", + "crossbeam-queue", + "crossbeam-skiplist", + "dashmap", + "datafusion", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-plan", + "either", + "fst", + "futures", + "humantime", + "itertools 0.14.0", + "lance-arrow", + "lance-bitpacking", + "lance-core", + "lance-datafusion", + "lance-encoding", + "lance-file", + "lance-index", + "lance-io", + "lance-linalg", + "lance-namespace", + "lance-select", + "lance-table", + "lance-tokenizer", + "log", + "moka", + "object_store 0.14.2", + "permutation", + "pin-project", + "prost", + "prost-build", + "prost-types", + "rand 0.9.5", + "rayon", + "roaring", + "rustc-hash", + "semver", + "serde", + "serde_json", + "snafu", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "lance-arrow" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab6e64549f9ad9a6380a35b56d7dca612555a6272c07b3abca84e59ded6f463" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "arrow-select", + "bytemuck", + "bytes", + "futures", + "getrandom 0.2.17", + "half", + "jsonb", + "num-traits", + "rand 0.9.5", +] + +[[package]] +name = "lance-arrow-scalar" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-row", + "arrow-schema", + "half", +] + +[[package]] +name = "lance-arrow-stats" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f" +dependencies = [ + "arrow-array", + "arrow-schema", + "half", + "lance-arrow-scalar", +] + +[[package]] +name = "lance-bitpacking" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38984429670b348e62eda38315a6d01a930c5c5e8b85bbfd8b998e7d1fe6a257" +dependencies = [ + "arrayref", + "crunchy", + "paste", + "seq-macro", +] + +[[package]] +name = "lance-core" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e377ee5bc5118efe5c321ecff0a07ac8f56db6bed16e31304e7d876ddcd96c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "async-trait", + "blake3", + "bytes", + "datafusion-common", + "datafusion-sql", + "futures", + "lance-arrow", + "lance-derive", + "libc", + "libm", + "log", + "moka", + "num_cpus", + "object_store 0.14.2", + "pin-project", + "prost", + "quick_cache", + "rand 0.9.5", + "roaring", + "serde_json", + "snafu", + "tempfile", + "tokio", + "tokio-util", + "tracing", + "twox-hash", + "url", +] + +[[package]] +name = "lance-datafusion" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7be43992965983ed8238a969b36239093102fdfe6c3dfcaa0007eeb17195ca34" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-trait", + "chrono", + "datafusion", + "datafusion-common", + "datafusion-functions", + "datafusion-physical-expr", + "futures", + "half", + "jsonb", + "lance-arrow", + "lance-core", + "log", + "pin-project", + "prost", + "prost-build", + "tokio", + "tracing", +] + +[[package]] +name = "lance-derive" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847c0e4bf23d21967fe79473d8001309c7351895afb347e1a997d5fadca59fd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "lance-encoding" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c04a91e545b505b5e6ec98854c7eadc56689cbc072f138dd26805ef0095e81a" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "bytemuck", + "byteorder", + "bytes", + "fsst", + "futures", + "hex", + "hyperloglogplus", + "itertools 0.14.0", + "lance-arrow", + "lance-bitpacking", + "lance-core", + "log", + "lz4", + "num-traits", + "prost", + "prost-build", + "tokio", + "tracing", + "xxhash-rust", + "zstd", +] + +[[package]] +name = "lance-file" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0393605ef4f2bdd9894950de6c68fd2ca99a1de76d743e52e001177ccd9994f" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "byteorder", + "bytes", + "datafusion-common", + "futures", + "lance-arrow", + "lance-core", + "lance-encoding", + "lance-io", + "log", + "num-traits", + "object_store 0.14.2", + "prost", + "prost-build", + "prost-types", + "serde", + "tokio", + "tracing", +] + +[[package]] +name = "lance-index" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b5e0ccd1b4bffdccbb3b5c989e62eb545587d84b0785da50eda5a4d87d0a5d" +dependencies = [ + "arc-swap", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-channel", + "async-recursion", + "async-trait", + "bitvec", + "bytes", + "crossbeam-queue", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "dirs", + "fst", + "futures", + "half", + "hashbrown 0.16.1", + "hex", + "itertools 0.14.0", + "jsonb", + "lance-arrow", + "lance-arrow-stats", + "lance-bitpacking", + "lance-core", + "lance-datafusion", + "lance-encoding", + "lance-file", + "lance-index-core", + "lance-io", + "lance-linalg", + "lance-select", + "lance-table", + "lance-tokenizer", + "libsais-rs", + "log", + "ndarray", + "num-traits", + "object_store 0.14.2", + "prost", + "prost-build", + "prost-types", + "rand 0.9.5", + "rand_distr", + "rangemap", + "rayon", + "regex-syntax", + "roaring", + "serde", + "serde_json", + "smallvec", + "tempfile", + "tokio", + "tracing", + "twox-hash", +] + +[[package]] +name = "lance-index-core" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90986278ab82f1f1b1fe43eed060f547e0da0ad98a1038785f055e407fca3db" +dependencies = [ + "arrow-array", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "datafusion", + "datafusion-common", + "datafusion-expr", + "futures", + "lance-core", + "lance-io", + "lance-select", + "prost-types", + "roaring", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "lance-io" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224730479e781d24b81a0afecfe230b5b21a86a4f767ba7655b974402d0c67c5" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "async-trait", + "aws-config", + "aws-credential-types", + "byteorder", + "bytes", + "chrono", + "futures", + "http", + "io-uring", + "lance-core", + "lance-namespace", + "log", + "moka", + "object_store 0.14.2", + "object_store_opendal", + "opendal", + "path_abs", + "pin-project", + "prost", + "rand 0.9.5", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "lance-linalg" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84e97d99cab96a320119f2be2c1ccbc4b69e5190c152b9bac96ad902b4409efe" +dependencies = [ + "arrow-array", + "arrow-schema", + "cc", + "half", + "lance-arrow", + "lance-core", + "num-traits", + "rayon", +] + +[[package]] +name = "lance-namespace" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eedb0930505992009d9a1b9aefc2e9fde6026fcc1b345211e2a609dace5be022" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "lance-core", + "lance-namespace-reqwest-client", + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "lance-namespace-reqwest-client" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "828f291764feda98898452bea4ce3756b73e5d5c7c67b62edda27456b7cc72c1" +dependencies = [ + "reqwest 0.12.28", + "serde", + "serde_json", + "serde_repr", + "serde_with", + "url", +] + +[[package]] +name = "lance-select" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cd00ec21363c9449f10409bf5b56e914b807dd9110745f0d616f65c16188ddb" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "byteorder", + "itertools 0.14.0", + "lance-core", + "roaring", + "tracing", +] + +[[package]] +name = "lance-table" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b770bac5033a6b3f11b3e3ffb05f8bce2cfc004ecb7992354af7627fd278747" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ipc", + "arrow-schema", + "async-trait", + "blake3", + "byteorder", + "bytes", + "chrono", + "futures", + "lance-arrow", + "lance-core", + "lance-file", + "lance-io", + "lance-select", + "log", + "object_store 0.14.2", + "prost", + "prost-build", + "prost-types", + "rand 0.9.5", + "rangemap", + "roaring", + "semver", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "lance-tokenizer" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fec4342e22fa493efec9cc96584efe4cecbdc1ae218d6a40f47b16bd4f1ea13" +dependencies = [ + "frostem", + "icu_segmenter", + "serde", + "stop-words", + "unicode-normalization", +] + +[[package]] +name = "lance_ladybug_ffi" +version = "0.1.0" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "futures", + "lance", + "lance-core", + "tokio", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin 0.9.9", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + +[[package]] +name = "libsais-rs" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00777f770949ecbe5524eb6630699a501c9c27e4d18493705a7dad49c9bdfbd1" +dependencies = [ + "rayon", +] + +[[package]] +name = "link-section" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" + +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "num_cpus", + "once_cell", + "rawpointer", + "thread-tree", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "objc2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.2", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.2", + "objc2", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-open-directory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb82bed227edf5201dfedf072bba4015a33d3d4a98519837295a90f0a23f676d" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "object_store" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64 0.23.1", + "bytes", + "chrono", + "crc-fast", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools 0.15.0", + "md-5", + "nix", + "parking_lot", + "percent-encoding", + "quick-xml 0.41.0", + "rand 0.10.3", + "reqwest 0.13.5", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", + "windows-sys 0.61.2", +] + +[[package]] +name = "object_store_opendal" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd9da6291f93eb19b7987dac17617de0131760a6ba97a4cd8d6b1a07fa89783" +dependencies = [ + "async-trait", + "asyncband", + "bytes", + "chrono", + "futures", + "object_store 0.14.2", + "opendal", + "pin-project", + "tokio", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "opendal" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46081824c74bac9f68f0bc4408077f77b0e315c006514719c97af1c2227ac2d2" +dependencies = [ + "ctor", + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-azblob", + "opendal-service-azdls", + "opendal-service-gcs", + "opendal-service-hf", + "opendal-service-oss", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d5a635dfea36a7a0c3bbb2ca34c97af17f98dd69b0cdf31a1d436a346e0a12" +dependencies = [ + "anyhow", + "asyncband", + "base64 0.23.1", + "bytes", + "futures", + "http", + "jiff", + "log", + "md-5", + "percent-encoding", + "quick-xml 0.42.0", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc025e163c43993e30cf1829a30b791934e5b0f7cd4d7e03acc902a9efb7e8f5" +dependencies = [ + "bytes", + "futures", + "http", + "http-body", + "opendal-core", + "reqwest 0.13.5", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06b44bdf694cccb7d90515b3eac6623975a2813abf4f5c438b4949a329d6a45b" +dependencies = [ + "asyncband", + "futures", + "http", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520d3de5d246c82a246c431ef0f52b8d2f091c7c49a3fd7a6f0ff519b1e7725f" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a02c59479b047878f4a0f269f23c847f24d67d1ff19f2b20fdd5a3bceefd451" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afffa0cf8a6690b3b44cc510816ba675638eb236699a7d449e44ac3344bafd36" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-azblob" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b674507cd2f764f6b40d5f88d284bb293c519d76e8aa0103987f970da412eeac" +dependencies = [ + "base64 0.23.1", + "bytes", + "http", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.42.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "sha2 0.11.1", + "uuid", +] + +[[package]] +name = "opendal-service-azdls" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afeb287cf2b03ab8d99d77a8e9687e908eb7898e2da1af6ccd00590c7095cae6" +dependencies = [ + "asyncband", + "base64 0.23.1", + "bytes", + "http", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.42.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdf9c9d0bfd1fa3638eec485c7f3448252cbb5a81774a2aa7abadc3e4f1ea43e" +dependencies = [ + "http", + "opendal-core", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e5197fce0cd5929db9129a7e71e503841aa20bea4f1dcc54ab097a5f3652e9" +dependencies = [ + "async-trait", + "bytes", + "http", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.42.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", + "serde", + "serde_json", + "tokio", + "uuid", +] + +[[package]] +name = "opendal-service-hf" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32c1a4076d933180b66b57a0f305e48f2de1a72b50648114cb9943a8bc9e28b8" +dependencies = [ + "asyncband", + "bytes", + "hf-xet", + "http", + "log", + "opendal-core", + "percent-encoding", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-oss" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e973fcebfa413432c024c41d65fad8034b22c065250c587e0c3a70f1a27eb6" +dependencies = [ + "bytes", + "http", + "log", + "opendal-core", + "quick-xml 0.42.0", + "reqsign-aliyun-oss", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", +] + +[[package]] +name = "opendal-service-s3" +version = "0.59.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0932321e27c712a30ef72db56b12f31e41e9f0d2d3841d63a91bc8075cd8f0ef" +dependencies = [ + "base64 0.23.1", + "bytes", + "crc-fast", + "http", + "log", + "md-5", + "opendal-core", + "quick-xml 0.42.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", +] + +[[package]] +name = "openssl-probe" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ordered-float" +version = "5.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f59006ce1e204c30956638535b74d317bca99477b9356dfaec54c37f6f600ab" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +dependencies = [ + "memchr", +] + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "path_abs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3" +dependencies = [ + "serde", + "serde_derive", + "std_prelude", + "stfu8", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "permutation" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df202b0b0f5b8e389955afd5f27b007b00fb948162953f1db9c70d2c7e3157d7" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap 2.14.2", + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "serde_core", + "writeable", + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.5", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rangemap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redb" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "redox_users" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" +dependencies = [ + "libredox", + "thiserror", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqsign-aliyun-oss" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cf3d65d37ac910073d921c1cc58c3e4e6c963795c6062eee196e6d19054dde" +dependencies = [ + "anyhow", + "form_urlencoded", + "http", + "log", + "percent-encoding", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", +] + +[[package]] +name = "reqsign-aws-core" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" +dependencies = [ + "bytes", + "form_urlencoded", + "hex", + "http", + "log", + "percent-encoding", + "quick-xml 0.42.0", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +dependencies = [ + "bytes", + "http", + "log", + "quick-xml 0.42.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-azure-storage" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7260e6c1d89422ba62d624a32c491b561386fb364ac4bf0959c740e34b251a2e" +dependencies = [ + "anyhow", + "asyncband", + "base64 0.23.1", + "bytes", + "form_urlencoded", + "http", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-core" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" +dependencies = [ + "anyhow", + "asyncband", + "base64 0.23.1", + "bytes", + "hex", + "hmac 0.13.0", + "http", + "jiff", + "log", + "percent-encoding", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719e55931cc81b8dcd69f95f0a2def55617cb76cb3aa2f8c439e10682e46fa10" +dependencies = [ + "bytes", + "form_urlencoded", + "http", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.2", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + +[[package]] +name = "reqwest-middleware" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest 0.13.5", + "thiserror", + "tower-service", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "safe-transmute" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.2", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "base64 0.23.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.2", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "bstr", + "dirs", + "os_str_bytes", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + +[[package]] +name = "std_prelude" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe" + +[[package]] +name = "stfu8" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978" + +[[package]] +name = "stop-words" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ed527ae13a020ac2404e32746ecee62d632aafe64f8e0df2bb3545ae96f6d1" +dependencies = [ + "serde_json", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "sysinfo" +version = "0.39.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2071df9448915b71c4fe6d25deaf1c22f12bd234f01540b77312bb8e41361e6" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "objc2-open-directory", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.2", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "thread-tree" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbd370cb847953a25954d9f63e14824a36113f8c72eecf6eccef5dc4b45d630" +dependencies = [ + "crossbeam-channel", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "serde_core", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tokio-retry" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" +dependencies = [ + "pin-project-lite", + "rand 0.10.3", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio_with_wasm" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e40fbbbd95441133fe9483f522db15dbfd26dc636164ebd8f2dd28759a6aa6" +dependencies = [ + "js-sys", + "tokio", + "tokio_with_wasm_proc", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "tokio_with_wasm_proc" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01145a2c788d6aae4cd653afec1e8332534d7d783d01897cefcafe4428de992" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags 2.13.2", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror", + "time", + "tracing-subscriber", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" +dependencies = [ + "rand 0.10.3", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "typewit" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" + +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xet-client" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15500e5fc9e17be15224f9e34cb07d41fb2805d9dbbf7bd3507c982fd1eb3efd" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "crc32fast", + "futures", + "http", + "hyper", + "more-asserts", + "rand 0.10.3", + "redb", + "reqwest 0.13.5", + "reqwest-middleware", + "serde", + "serde_json", + "serde_repr", + "statrs", + "tempfile", + "thiserror", + "tokio", + "tokio-retry", + "tokio_with_wasm", + "tracing", + "url", + "urlencoding", + "uuid", + "web-time", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-core-structures" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5080d5803c8bf15fa56ef934050a46b225801bad3ea4cb8bb3c46c5c37976516" +dependencies = [ + "async-trait", + "base64 0.22.1", + "blake3", + "bytemuck", + "bytes", + "countio", + "futures", + "futures-util", + "getrandom 0.4.3", + "heapify", + "itertools 0.14.0", + "lz4_flex", + "more-asserts", + "rand 0.10.3", + "regex", + "safe-transmute", + "serde", + "static_assertions", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "uuid", + "web-time", + "xet-runtime", +] + +[[package]] +name = "xet-data" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1287a1657fb67de846030f6605a9960dcdad401c467b490657e6e909d2e0d1c0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "gearhash", + "http", + "itertools 0.14.0", + "more-asserts", + "rand 0.10.3", + "serde", + "serde_json", + "sha2 0.11.1", + "tempfile", + "thiserror", + "tokio", + "tokio-util", + "tokio_with_wasm", + "tracing", + "url", + "uuid", + "web-time", + "xet-client", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-runtime" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921953c885ce83e02b21e8d26444b75c78c9e83076699ced08d5c6fc1cdb48ac" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "colored", + "const-str", + "ctor", + "dirs", + "futures", + "git-version", + "humantime", + "konst", + "libc", + "more-asserts", + "oneshot", + "pin-project", + "rand 0.10.3", + "reqwest 0.13.5", + "serde", + "serde_json", + "shellexpand", + "sysinfo", + "thiserror", + "tokio", + "tokio-util", + "tokio_with_wasm", + "tracing", + "tracing-appender", + "tracing-subscriber", + "web-time", + "whoami", + "winapi", +] + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "xxhash-rust" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "serde", + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/lance/rust/Cargo.toml b/lance/rust/Cargo.toml new file mode 100644 index 00000000..c7b1a934 --- /dev/null +++ b/lance/rust/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "lance_ladybug_ffi" +version = "0.1.0" +publish = false +edition = "2021" + +[lib] +name = "lance_ladybug_ffi" +crate-type = ["staticlib"] + +[profile.release] +lto = "thin" +strip = true +panic = "abort" + +[profile.dev] +panic = "abort" + +[dependencies] +# Same Lance/Arrow versions as lance-duckdb main: the reader half of that +# engine, without the DuckDBQFII/DataFusion query layer. +lance = { version = "13.0.0", default-features = false, features = ["aws", "azure", "gcp", "oss", "huggingface"] } +lance-core = "13.0.0" +arrow = { version = "58.0.0", features = ["ffi"] } +arrow-array = "58.0.0" +arrow-schema = "58.0.0" +futures = "0.3" +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } diff --git a/lance/rust/examples/gen_fixtures.rs b/lance/rust/examples/gen_fixtures.rs new file mode 100644 index 00000000..e64db909 --- /dev/null +++ b/lance/rust/examples/gen_fixtures.rs @@ -0,0 +1,152 @@ +//! Regenerates the Lance test fixtures under `test/lance_data/`. +//! +//! Content mirrors `dataset/ice-disk-test` (the icebug-disk fixtures) +//! row-for-row; every dataset is additionally stamped with +//! `lance_version = v1` schema metadata (plus one `v999` dataset for the +//! version-mismatch error test). Run via +//! `extension/lance/scripts/gen_test_data.sh`. +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::{Int32Array, Int64Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field, Schema}; +use lance::dataset::{WriteMode, WriteParams}; +use lance::Dataset; + +fn schema_with_version(fields: Vec, version: &str) -> Arc { + Arc::new(Schema::new_with_metadata( + fields, + HashMap::from([("lance_version".to_string(), version.to_string())]), + )) +} + +async fn write_dataset(out_dir: &str, name: &str, schema: Arc, batch: RecordBatch) { + let uri = format!("{out_dir}/{name}"); + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + let params = WriteParams { + mode: WriteMode::Overwrite, + ..Default::default() + }; + Dataset::write(reader, &uri, Some(params)) + .await + .unwrap_or_else(|err| panic!("failed to write {uri}: {err}")); + println!("wrote {uri}"); +} + +fn int32_col(values: Vec) -> Arc { + Arc::new(Int32Array::from(values)) +} + +fn int64_col(values: Vec) -> Arc { + Arc::new(Int64Array::from(values)) +} + +fn str_col(values: Vec<&str>) -> Arc { + Arc::new(StringArray::from(values)) +} + +#[tokio::main] +async fn main() { + let out_dir = std::env::args() + .nth(1) + .expect("usage: gen_fixtures "); + std::fs::create_dir_all(&out_dir).unwrap(); + + // nodes_user(id INT32, name VARCHAR, age BIGINT) + let schema = schema_with_version( + vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("age", DataType::Int64, false), + ], + "v1", + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + int32_col(vec![75, 100, 250, 300]), + str_col(vec!["Noura", "Adam", "Karissa", "Zhang"]), + int64_col(vec![25, 30, 40, 50]), + ], + ) + .unwrap(); + write_dataset(&out_dir, "nodes_user.lance", schema, batch).await; + + // nodes_city(id INT32, name VARCHAR, population BIGINT) + let schema = schema_with_version( + vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("population", DataType::Int64, false), + ], + "v1", + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + int32_col(vec![500, 600, 700]), + str_col(vec!["Guelph", "Kitchener", "Waterloo"]), + int64_col(vec![75000, 200000, 150000]), + ], + ) + .unwrap(); + write_dataset(&out_dir, "nodes_city.lance", schema, batch).await; + + // indices_follows(target BIGINT, since INT32), CSR-sorted by source. + let schema = schema_with_version( + vec![ + Field::new("target", DataType::Int64, false), + Field::new("since", DataType::Int32, false), + ], + "v1", + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + int64_col(vec![1, 1, 2, 3, 1, 3, 0]), + int32_col(vec![2023, 2023, 2020, 2020, 2022, 2021, 2022]), + ], + ) + .unwrap(); + write_dataset(&out_dir, "indices_follows.lance", schema, batch).await; + + // indptr_follows(ptr BIGINT) + let schema = schema_with_version(vec![Field::new("ptr", DataType::Int64, false)], "v1"); + let batch = RecordBatch::try_new(schema.clone(), vec![int64_col(vec![0, 1, 4, 6, 7])]).unwrap(); + write_dataset(&out_dir, "indptr_follows.lance", schema, batch).await; + + // indices_livesin(target BIGINT), no edge properties. + let schema = schema_with_version(vec![Field::new("target", DataType::Int64, false)], "v1"); + let batch = RecordBatch::try_new(schema.clone(), vec![int64_col(vec![0, 2, 2, 1])]).unwrap(); + write_dataset(&out_dir, "indices_livesin.lance", schema, batch).await; + + // indptr_livesin(ptr BIGINT) + let schema = schema_with_version(vec![Field::new("ptr", DataType::Int64, false)], "v1"); + let batch = RecordBatch::try_new(schema.clone(), vec![int64_col(vec![0, 1, 2, 3, 4])]).unwrap(); + write_dataset(&out_dir, "indptr_livesin.lance", schema, batch).await; + + // rels_follows_flat(src BIGINT, dst BIGINT, since INT32) + let schema = schema_with_version( + vec![ + Field::new("src", DataType::Int64, false), + Field::new("dst", DataType::Int64, false), + Field::new("since", DataType::Int32, false), + ], + "v1", + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + int64_col(vec![0, 1, 1, 1, 2, 2, 3]), + int64_col(vec![1, 1, 2, 3, 1, 3, 0]), + int32_col(vec![2023, 2023, 2020, 2020, 2022, 2021, 2022]), + ], + ) + .unwrap(); + write_dataset(&out_dir, "rels_follows_flat.lance", schema, batch).await; + + // nodes_wrongversion: stamped v999 for the version-mismatch error test. + let schema = schema_with_version(vec![Field::new("id", DataType::Int32, false)], "v999"); + let batch = RecordBatch::try_new(schema.clone(), vec![int32_col(vec![1])]).unwrap(); + write_dataset(&out_dir, "nodes_wrongversion.lance", schema, batch).await; +} diff --git a/lance/rust/src/error.rs b/lance/rust/src/error.rs new file mode 100644 index 00000000..e5343f25 --- /dev/null +++ b/lance/rust/src/error.rs @@ -0,0 +1,70 @@ +use std::cell::RefCell; +use std::ffi::{c_char, CString}; +use std::ptr; + +#[repr(i32)] +#[derive(Clone, Copy, Debug)] +pub enum ErrorCode { + InvalidArgument = 1, + Utf8 = 2, + Runtime = 3, + DatasetOpen = 4, + DatasetCountRows = 5, + SchemaExport = 6, + ScanCreate = 7, + ScanNext = 8, + BatchExport = 9, +} + +struct LastError { + code: i32, + message: CString, +} + +thread_local! { + static LAST_ERROR: RefCell> = const { RefCell::new(None) }; +} + +fn sanitize_message(message: &str) -> CString { + match CString::new(message) { + Ok(v) => v, + Err(_) => CString::new(message.replace('\0', "\\0")) + .unwrap_or_else(|_| CString::new("invalid error message").unwrap()), + } +} + +pub fn clear_last_error() { + LAST_ERROR.with(|e| { + *e.borrow_mut() = None; + }); +} + +pub fn set_last_error(code: ErrorCode, message: impl AsRef) { + let code = code as i32; + let message = sanitize_message(message.as_ref()); + LAST_ERROR.with(|e| { + *e.borrow_mut() = Some(LastError { code, message }); + }); +} + +#[no_mangle] +pub extern "C" fn lance_last_error_code() -> i32 { + LAST_ERROR.with(|e| e.borrow().as_ref().map(|v| v.code).unwrap_or(0)) +} + +#[no_mangle] +pub extern "C" fn lance_last_error_message() -> *const c_char { + LAST_ERROR.with(|e| match e.borrow_mut().take() { + Some(err) => err.message.into_raw() as *const c_char, + None => ptr::null(), + }) +} + +#[no_mangle] +pub unsafe extern "C" fn lance_free_string(s: *const c_char) { + if !s.is_null() { + unsafe { + let _ = CString::from_raw(s as *mut c_char); + } + } +} diff --git a/lance/rust/src/ffi.rs b/lance/rust/src/ffi.rs new file mode 100644 index 00000000..91f4f677 --- /dev/null +++ b/lance/rust/src/ffi.rs @@ -0,0 +1,578 @@ +use std::ffi::{c_char, c_void, CStr, CString}; +use std::pin::Pin; +use std::ptr; +use std::sync::Arc; + +use arrow::array::{Array, RecordBatch, StructArray}; +use arrow::datatypes::DataType; +use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; +use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; +use futures::StreamExt; +use lance::dataset::builder::DatasetBuilder; +use lance::dataset::scanner::DatasetRecordBatchStream; +use lance::Dataset; +use lance_core::datatypes::{LogicalType as LanceLogicalType, Schema as LanceSchema}; +use tokio::runtime::Handle; + +use crate::error::{clear_last_error, set_last_error, ErrorCode}; +use crate::runtime; + +const VERSION_METADATA_KEY: &str = "lance_version"; + +/// Max rows per exported batch slice. Matches Ladybug's default +/// `DEFAULT_VECTOR_CAPACITY` (2^11); the C++ side converts each slice into a +/// fresh engine chunk, so slices must never exceed it. +const BATCH_SLICE_ROWS: usize = 2048; + +pub(crate) struct DatasetHandle { + dataset: Arc, +} + +pub(crate) struct StreamHandle { + handle: Handle, + stream: Pin>, + pending: Option, + pending_pos: usize, +} + +struct FfiError { + code: ErrorCode, + message: String, +} + +impl FfiError { + fn new(code: ErrorCode, message: String) -> Self { + Self { code, message } + } +} + +type FfiResult = Result; + +fn dataset_handle(dataset: *mut c_void) -> FfiResult<&'static DatasetHandle> { + if dataset.is_null() { + return Err(FfiError::new( + ErrorCode::InvalidArgument, + "dataset is null".to_string(), + )); + } + Ok(unsafe { &*(dataset as *const DatasetHandle) }) +} + +fn stream_handle_mut(stream: *mut c_void) -> FfiResult<&'static mut StreamHandle> { + if stream.is_null() { + return Err(FfiError::new( + ErrorCode::InvalidArgument, + "stream is null".to_string(), + )); + } + Ok(unsafe { &mut *(stream as *mut StreamHandle) }) +} + +fn batch_handle(batch: *mut c_void) -> FfiResult<&'static RecordBatch> { + if batch.is_null() { + return Err(FfiError::new( + ErrorCode::InvalidArgument, + "batch is null".to_string(), + )); + } + Ok(unsafe { &*(batch as *const RecordBatch) }) +} + +fn cstr_to_str<'a>(ptr: *const c_char, name: &str) -> FfiResult<&'a str> { + if ptr.is_null() { + return Err(FfiError::new( + ErrorCode::InvalidArgument, + format!("{name} is null"), + )); + } + unsafe { CStr::from_ptr(ptr) } + .to_str() + .map_err(|err| FfiError::new(ErrorCode::Utf8, format!("{name} utf8: {err}"))) +} + +fn cstr_array(ptr: *const *const c_char, len: usize, name: &str) -> FfiResult> { + if len == 0 { + return Ok(Vec::new()); + } + if ptr.is_null() { + return Err(FfiError::new( + ErrorCode::InvalidArgument, + format!("{name} is null with non-zero length"), + )); + } + let mut out = Vec::with_capacity(len); + for i in 0..len { + let item = unsafe { *ptr.add(i) }; + out.push(cstr_to_str(item, name)?.to_string()); + } + Ok(out) +} + +fn to_c_string(value: String) -> CString { + match CString::new(value) { + Ok(v) => v, + Err(_) => CString::new("invalid string").unwrap(), + } +} + +fn reject(lt: &LanceLogicalType, reason: &str) -> FfiError { + FfiError::new( + ErrorCode::SchemaExport, + format!("unsupported lance column type '{lt}': {reason}"), + ) +} + +/// Map one Lance schema field to its Arrow field. Leaf types convert through +/// Lance's own `TryFrom<&LogicalType>`; structural types recurse into +/// children. Anything else fails fast so values can never be silently +/// mistyped downstream. +fn lance_field_to_arrow(field: &lance_core::datatypes::Field) -> FfiResult { + let text = field.logical_type.to_string(); + let data_type = if text == "struct" { + let children = field + .children + .iter() + .map(lance_field_to_arrow) + .collect::>>()?; + DataType::Struct(children.into()) + } else if text == "list" + || text == "list.struct" + || text == "large_list" + || text == "large_list.struct" + { + let element = field.children.first().ok_or_else(|| { + FfiError::new( + ErrorCode::SchemaExport, + "list column has no element".to_string(), + ) + })?; + let element = lance_field_to_arrow(element)?; + if text.starts_with("large_list") { + DataType::LargeList(Arc::new(element)) + } else { + DataType::List(Arc::new(element)) + } + } else if text.starts_with("fixed_size_list:") { + let element = field.children.first().ok_or_else(|| { + FfiError::new( + ErrorCode::SchemaExport, + "fixed-size list column has no element".to_string(), + ) + })?; + let element = lance_field_to_arrow(element)?; + let width: i32 = text + .rsplit(':') + .next() + .and_then(|w| w.parse().ok()) + .ok_or_else(|| { + FfiError::new( + ErrorCode::SchemaExport, + format!("cannot parse fixed-size list width from '{text}'"), + ) + })?; + DataType::FixedSizeList(Arc::new(element), width) + } else if text == "map" { + return Err(reject( + &field.logical_type, + "map columns are not supported in v1; flatten them into top-level columns", + )); + } else if text.starts_with("timestamp:") { + // Timestamps with a zone would be silently mistyped downstream, and + // durations/time32/time64 have no exact counterpart either. + let tz_naive = text == "timestamp:s:-" + || text == "timestamp:ms:-" + || text == "timestamp:us:-" + || text == "timestamp:ns:-"; + if !tz_naive { + return Err(reject( + &field.logical_type, + "only timezone-naive timestamps are supported", + )); + } + DataType::try_from(&field.logical_type).map_err(|err| { + FfiError::new( + ErrorCode::SchemaExport, + format!("timestamp conversion: {err}"), + ) + })? + } else if text.starts_with("duration:") + || text.starts_with("time32:") + || text.starts_with("time64:") + { + return Err(reject( + &field.logical_type, + "only date/timestamp/interval temporal types are supported", + )); + } else { + DataType::try_from(&field.logical_type).map_err(|err| { + FfiError::new(ErrorCode::SchemaExport, format!("type conversion: {err}")) + })? + }; + Ok(ArrowField::new(&field.name, data_type, field.nullable)) +} + +fn lance_schema_to_arrow_schema(schema: &LanceSchema) -> FfiResult { + let fields = schema + .fields + .iter() + .map(lance_field_to_arrow) + .collect::>>()?; + Ok(ArrowSchema::new(fields)) +} + +#[no_mangle] +pub unsafe extern "C" fn lance_open_dataset(path: *const c_char) -> *mut c_void { + match open_dataset_inner(path) { + Ok(handle) => { + clear_last_error(); + Box::into_raw(Box::new(handle)) as *mut c_void + } + Err(err) => { + set_last_error(err.code, err.message); + ptr::null_mut() + } + } +} + +fn open_dataset_inner(path: *const c_char) -> FfiResult { + let path_str = cstr_to_str(path, "path")?; + // Opening pins the dataset handle to the latest committed version: every + // scan through the handle reads one stable snapshot. + let dataset = match runtime::block_on(DatasetBuilder::from_uri(path_str).load()) { + Ok(Ok(ds)) => Arc::new(ds), + Ok(Err(err)) => { + return Err(FfiError::new( + ErrorCode::DatasetOpen, + format!("dataset open '{path_str}': {err}"), + )) + } + Err(err) => return Err(FfiError::new(ErrorCode::Runtime, format!("runtime: {err}"))), + }; + Ok(DatasetHandle { dataset }) +} + +#[no_mangle] +pub unsafe extern "C" fn lance_close_dataset(dataset: *mut c_void) { + if !dataset.is_null() { + unsafe { + let _ = Box::from_raw(dataset as *mut DatasetHandle); + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn lance_dataset_count_rows(dataset: *mut c_void) -> i64 { + match dataset_count_rows_inner(dataset) { + Ok(v) => { + clear_last_error(); + v + } + Err(err) => { + set_last_error(err.code, err.message); + -1 + } + } +} + +fn dataset_count_rows_inner(dataset: *mut c_void) -> FfiResult { + let handle = dataset_handle(dataset)?; + let rows = match runtime::block_on(handle.dataset.count_rows(None)) { + Ok(Ok(rows)) => rows, + Ok(Err(err)) => { + return Err(FfiError::new( + ErrorCode::DatasetCountRows, + format!("dataset count_rows: {err}"), + )) + } + Err(err) => return Err(FfiError::new(ErrorCode::Runtime, format!("runtime: {err}"))), + }; + i64::try_from(rows).map_err(|_| { + FfiError::new( + ErrorCode::DatasetCountRows, + "row count overflow".to_string(), + ) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn lance_dataset_version(dataset: *mut c_void, out_version: *mut u64) -> i32 { + if out_version.is_null() { + set_last_error(ErrorCode::InvalidArgument, "out_version must not be null"); + return -1; + } + match dataset_handle(dataset) { + Ok(handle) => { + clear_last_error(); + unsafe { + *out_version = handle.dataset.manifest.version; + } + 0 + } + Err(err) => { + set_last_error(err.code, err.message); + -1 + } + } +} + +/// Export the dataset's Arrow schema (all columns, in dataset order). +#[no_mangle] +pub unsafe extern "C" fn lance_schema_to_arrow( + dataset: *mut c_void, + out_schema: *mut FFI_ArrowSchema, +) -> i32 { + match schema_to_arrow_inner(dataset, out_schema) { + Ok(()) => { + clear_last_error(); + 0 + } + Err(err) => { + set_last_error(err.code, err.message); + -1 + } + } +} + +fn schema_to_arrow_inner(dataset: *mut c_void, out_schema: *mut FFI_ArrowSchema) -> FfiResult<()> { + if out_schema.is_null() { + return Err(FfiError::new( + ErrorCode::InvalidArgument, + "out_schema is null".to_string(), + )); + } + let handle = dataset_handle(dataset)?; + let arrow_schema = lance_schema_to_arrow_schema(handle.dataset.schema())?; + let ffi_schema = FFI_ArrowSchema::try_from(&DataType::Struct(arrow_schema.fields().clone())) + .map_err(|err| FfiError::new(ErrorCode::SchemaExport, format!("schema export: {err}")))?; + unsafe { + std::ptr::write_unaligned(out_schema, ffi_schema); + } + Ok(()) +} + +/// Return the dataset's `lance_version` schema-metadata value, or an empty +/// string when the marker is absent. +#[no_mangle] +pub unsafe extern "C" fn lance_schema_version_string(dataset: *mut c_void) -> *const c_char { + match schema_version_string_inner(dataset) { + Ok(value) => { + clear_last_error(); + to_c_string(value).into_raw() as *const c_char + } + Err(err) => { + set_last_error(err.code, err.message); + ptr::null() + } + } +} + +fn schema_version_string_inner(dataset: *mut c_void) -> FfiResult { + let handle = dataset_handle(dataset)?; + Ok(handle + .dataset + .schema() + .metadata + .get(VERSION_METADATA_KEY) + .cloned() + .unwrap_or_default()) +} + +#[no_mangle] +pub unsafe extern "C" fn lance_scan_open( + dataset: *mut c_void, + columns: *const *const c_char, + columns_len: usize, + limit: i64, + offset: u64, +) -> *mut c_void { + match scan_open_inner(dataset, columns, columns_len, limit, offset) { + Ok(stream) => { + clear_last_error(); + Box::into_raw(Box::new(stream)) as *mut c_void + } + Err(err) => { + set_last_error(err.code, err.message); + ptr::null_mut() + } + } +} + +fn scan_open_inner( + dataset: *mut c_void, + columns: *const *const c_char, + columns_len: usize, + limit: i64, + offset: u64, +) -> FfiResult { + let handle = dataset_handle(dataset)?; + if limit < -1 { + return Err(FfiError::new( + ErrorCode::ScanCreate, + "limit must be >= -1".to_string(), + )); + } + let projection = cstr_array(columns, columns_len, "columns")?; + + let mut scan = handle.dataset.scan(); + if !projection.is_empty() { + scan.project(&projection) + .map_err(|err| FfiError::new(ErrorCode::ScanCreate, format!("scan project: {err}")))?; + } + scan.blob_handling(lance_core::datatypes::BlobHandling::AllBinary); + if limit != -1 || offset != 0 { + let limit_opt = if limit == -1 { None } else { Some(limit) }; + let offset_opt = + if offset == 0 { + None + } else { + Some(i64::try_from(offset).map_err(|_| { + FfiError::new(ErrorCode::ScanCreate, "offset overflow".to_string()) + })?) + }; + scan.limit(limit_opt, offset_opt) + .map_err(|err| FfiError::new(ErrorCode::ScanCreate, format!("scan limit: {err}")))?; + } + // In-order scans: CSR range reads map batch positions back to source rows, + // which requires dataset order. + scan.scan_in_order(true); + + let rt_handle = runtime::handle() + .map_err(|err| FfiError::new(ErrorCode::Runtime, format!("runtime: {err}")))?; + let stream = rt_handle + .block_on(async { scan.try_into_stream().await }) + .map_err(|err| FfiError::new(ErrorCode::ScanCreate, format!("stream create: {err}")))?; + Ok(StreamHandle { + handle: rt_handle, + stream: Box::pin(stream), + pending: None, + pending_pos: 0, + }) +} + +/// Fetch the next batch slice (at most BATCH_SLICE_ROWS rows). +/// Returns 0 with a batch, 1 when exhausted, -1 on error. +#[no_mangle] +pub unsafe extern "C" fn lance_scan_next(stream: *mut c_void, out_batch: *mut *mut c_void) -> i32 { + match scan_next_inner(stream, out_batch) { + Ok(v) => { + clear_last_error(); + v + } + Err(err) => { + set_last_error(err.code, err.message); + -1 + } + } +} + +fn scan_next_inner(stream: *mut c_void, out_batch: *mut *mut c_void) -> FfiResult { + if out_batch.is_null() { + return Err(FfiError::new( + ErrorCode::ScanNext, + "out_batch is null".to_string(), + )); + } + unsafe { + ptr::write_unaligned(out_batch, ptr::null_mut()); + } + let handle = stream_handle_mut(stream)?; + + loop { + if let Some(batch) = handle.pending.as_ref() { + let remaining = batch.num_rows().saturating_sub(handle.pending_pos); + if remaining > 0 { + let len = remaining.min(BATCH_SLICE_ROWS); + let slice = batch.slice(handle.pending_pos, len); + handle.pending_pos += len; + if handle.pending_pos >= batch.num_rows() { + handle.pending = None; + handle.pending_pos = 0; + } + let out = Box::into_raw(Box::new(slice)) as *mut c_void; + unsafe { + ptr::write_unaligned(out_batch, out); + } + return Ok(0); + } + handle.pending = None; + handle.pending_pos = 0; + } + let next = handle.handle.block_on(async { handle.stream.next().await }); + match next { + Some(Ok(batch)) => { + if batch.num_rows() == 0 { + continue; + } + handle.pending = Some(batch); + handle.pending_pos = 0; + } + Some(Err(err)) => { + return Err(FfiError::new( + ErrorCode::ScanNext, + format!("stream next: {err}"), + )) + } + None => return Ok(1), + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn lance_close_scan(stream: *mut c_void) { + if !stream.is_null() { + unsafe { + let _ = Box::from_raw(stream as *mut StreamHandle); + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn lance_batch_to_arrow( + batch: *mut c_void, + out_array: *mut FFI_ArrowArray, + out_schema: *mut FFI_ArrowSchema, +) -> i32 { + match batch_to_arrow_inner(batch, out_array, out_schema) { + Ok(()) => { + clear_last_error(); + 0 + } + Err(err) => { + set_last_error(err.code, err.message); + -1 + } + } +} + +fn batch_to_arrow_inner( + batch: *mut c_void, + out_array: *mut FFI_ArrowArray, + out_schema: *mut FFI_ArrowSchema, +) -> FfiResult<()> { + if out_array.is_null() || out_schema.is_null() { + return Err(FfiError::new( + ErrorCode::BatchExport, + "out_array/out_schema is null".to_string(), + )); + } + let batch = batch_handle(batch)?; + let struct_array = StructArray::from(batch.clone()); + let data = struct_array.to_data(); + let array = FFI_ArrowArray::new(&data); + let schema = FFI_ArrowSchema::try_from(data.data_type()) + .map_err(|err| FfiError::new(ErrorCode::BatchExport, format!("batch export: {err}")))?; + unsafe { + std::ptr::write_unaligned(out_array, array); + std::ptr::write_unaligned(out_schema, schema); + } + Ok(()) +} + +#[no_mangle] +pub unsafe extern "C" fn lance_free_batch(batch: *mut c_void) { + if !batch.is_null() { + unsafe { + let _ = Box::from_raw(batch as *mut RecordBatch); + } + } +} diff --git a/lance/rust/src/lib.rs b/lance/rust/src/lib.rs new file mode 100644 index 00000000..2e9b41dd --- /dev/null +++ b/lance/rust/src/lib.rs @@ -0,0 +1,5 @@ +#![allow(clippy::missing_safety_doc)] + +mod error; +mod ffi; +mod runtime; diff --git a/lance/rust/src/runtime.rs b/lance/rust/src/runtime.rs new file mode 100644 index 00000000..2c6affc9 --- /dev/null +++ b/lance/rust/src/runtime.rs @@ -0,0 +1,22 @@ +use std::future::Future; +use std::io; +use std::sync::OnceLock; + +use tokio::runtime::{Handle, Runtime}; + +static RUNTIME: OnceLock> = OnceLock::new(); + +pub fn runtime() -> Result<&'static Runtime, io::Error> { + match RUNTIME.get_or_init(Runtime::new) { + Ok(rt) => Ok(rt), + Err(err) => Err(io::Error::new(err.kind(), err.to_string())), + } +} + +pub fn handle() -> Result { + Ok(runtime()?.handle().clone()) +} + +pub fn block_on(future: F) -> Result { + Ok(runtime()?.block_on(future)) +} diff --git a/lance/scripts/gen_test_data.sh b/lance/scripts/gen_test_data.sh index 4f6d1cd8..b795e782 100755 --- a/lance/scripts/gen_test_data.sh +++ b/lance/scripts/gen_test_data.sh @@ -1,9 +1,9 @@ #!/bin/bash # Regenerates the Lance test datasets under extension/lance/test/lance_data/. -# Uses the DuckDB CLI with its lance core extension (the same engine the lance -# extension reads through), so the files are always readable by the extension. -# Content mirrors dataset/ice-disk-test (the icebug-disk fixtures) row-for-row, -# letting the e2e expectations mirror test/test_files/ice_disk/. +# Uses the extension's own Rust fixture generator (same Lance version as the +# reader), so files are always in a readable format. Content mirrors +# dataset/ice-disk-test (the icebug-disk fixtures) row-for-row, letting the +# e2e expectations mirror test/test_files/ice_disk/. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -11,67 +11,8 @@ OUT_DIR="${SCRIPT_DIR}/../test/lance_data" rm -rf "${OUT_DIR}" mkdir -p "${OUT_DIR}" -duckdb -unsigned :memory: < - PARENT_SCOPE) diff --git a/lance/src/connector/lance_connector.cpp b/lance/src/connector/lance_connector.cpp deleted file mode 100644 index 503f3a9d..00000000 --- a/lance/src/connector/lance_connector.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "connector/lance_connector.h" - -#include "common/exception/runtime.h" - -namespace lbug { -namespace lance_extension { - -void LanceConnector::connect(const std::string& /*dbPath*/, const std::string& /*catalogName*/, - const std::string& /*schemaName*/, main::ClientContext* context) { - // Creates an in-memory duckdb instance, then loads the lance extension (the - // same engine as lance-duckdb: a DuckDB core extension) and httpfs for - // object-store URIs. - instance = std::make_unique(nullptr); - connection = std::make_unique(*instance); - executeQuery("INSTALL lance;"); - executeQuery("LOAD lance;"); - executeQuery("INSTALL httpfs;"); - executeQuery("LOAD httpfs;"); - initRemoteFSSecrets(context); -} - -std::unique_ptr LanceConnector::newConnection(main::ClientContext* context) { - DASSERT(instance != nullptr); - auto conn = std::make_unique(*instance); - // LOAD is idempotent, and object-store secrets configured by connect() live on the - // shared instance, so every connection inherits them. - auto result = conn->Query("LOAD lance;"); - if (result->HasError()) { - throw common::RuntimeException{ - "Failed to load lance extension on scan connection: " + result->GetError()}; - } - return conn; -} - -std::unique_ptr LanceConnector::executeOn(duckdb::Connection& connection, - const std::string& query) const { - auto result = connection.Query(query); - if (result->HasError()) { - throw common::Exception{result->GetError()}; - } - return result; -} - -std::string LanceConnector::escapeLiteral(const std::string& value) { - std::string out; - out.reserve(value.size()); - for (char c : value) { - if (c == '\'') { - out += "''"; - } else { - out += c; - } - } - return out; -} - -std::string LanceConnector::quoteIdent(const std::string& ident) { - std::string out = "\""; - for (char c : ident) { - if (c == '"') { - out += "\"\""; - } else { - out += c; - } - } - out += "\""; - return out; -} - -std::string LanceConnector::scanAllQuery(const std::string& datasetPath) { - return "SELECT * FROM '" + escapeLiteral(datasetPath) + "'"; -} - -std::string LanceConnector::countRowsQuery(const std::string& datasetPath) { - return "SELECT COUNT(*) FROM '" + escapeLiteral(datasetPath) + "'"; -} - -std::string LanceConnector::schemaQuery(const std::string& datasetPath) { - return scanAllQuery(datasetPath) + " LIMIT 0"; -} - -} // namespace lance_extension -} // namespace lbug diff --git a/lance/src/function/CMakeLists.txt b/lance/src/function/CMakeLists.txt index 6d9ec8d2..215a1870 100644 --- a/lance/src/function/CMakeLists.txt +++ b/lance/src/function/CMakeLists.txt @@ -1,9 +1,6 @@ add_library(lbug_lance_function OBJECT - lance_scan.cpp - ${PROJECT_SOURCE_DIR}/extension/duckdb/src/connector/duckdb_result_converter.cpp - ${PROJECT_SOURCE_DIR}/extension/duckdb/src/connector/duckdb_type_converter.cpp - ${PROJECT_SOURCE_DIR}/extension/duckdb/src/function/duckdb_scan.cpp) + lance_scan.cpp) set(LANCE_EXTENSION_OBJECT_FILES ${LANCE_EXTENSION_OBJECT_FILES} $ diff --git a/lance/src/function/lance_scan.cpp b/lance/src/function/lance_scan.cpp index bf7291de..acdb000f 100644 --- a/lance/src/function/lance_scan.cpp +++ b/lance/src/function/lance_scan.cpp @@ -1,6 +1,13 @@ #include "function/lance_scan.h" -#include +#include + +#include "common/data_chunk/data_chunk.h" +#include "common/exception/binder.h" +#include "common/vector/value_vector.h" +#include "function/table/bind_input.h" +#include "processor/execution_context.h" +#include "storage/buffer_manager/memory_manager.h" namespace lbug { namespace lance_extension { @@ -14,56 +21,75 @@ static std::unique_ptr bindFunc(main::ClientContext* context, const auto* scanInput = input->extraInput ? input->extraInput->constPtrCast() : nullptr; - auto connector = std::make_shared(); - connector->connect("" /* inMemDB */, "" /* defaultCatalogName */, "" /* defaultSchemaName */, - context); - // DuckDB's lance extension resolves '.lance' through its replacement scan. - std::string query = std::format("SELECT * FROM '{}'", - LanceConnector::escapeLiteral(input->getLiteralVal(0))); - auto result = connector->executeQuery(query + " LIMIT 1"); + const auto datasetPath = input->getLiteralVal(0); + LanceDataset dataset{datasetPath}; + const auto schema = dataset.schema(); std::vector returnTypes; std::vector returnColumnNames = scanInput ? scanInput->expectedColumnNames : std::vector{}; - for (auto type : result->GetTypes()) { - returnTypes.push_back( - duckdb_extension::DuckDBTypeConverter::convertDuckDBType(type.ToString())); - } - if (returnColumnNames.empty()) { - for (auto name : result->GetNames()) { - returnColumnNames.push_back(name.GetIdentifierName()); + for (auto& col : schema) { + returnColumnNames.push_back(col.name); + returnTypes.push_back(col.ladybugType.copy()); + } + } else { + for (auto& name : returnColumnNames) { + auto it = std::find_if(schema.begin(), schema.end(), + [&](const LanceDatasetColumn& col) { return col.name == name; }); + if (it == schema.end()) { + throw BinderException{ + "Lance dataset '" + datasetPath + "' has no column '" + name + "'"}; + } + returnTypes.push_back(it->ladybugType.copy()); } } DASSERT(returnTypes.size() == returnColumnNames.size()); returnColumnNames = TableFunction::extractYieldVariables(returnColumnNames, input->yieldVariables); auto columns = input->binder->createVariables(returnColumnNames, returnTypes); - return std::make_unique(std::move(query), connector, - duckdb_extension::DuckDBResultConverter{returnTypes}, columns, context); + return std::make_unique(datasetPath, std::move(returnColumnNames), + std::move(returnTypes), std::move(columns), context); } std::unique_ptr initLanceScanSharedState( const TableFuncInitSharedStateInput& input) { - auto lanceScanBindData = input.bindData->constPtrCast(); - auto queryResult = lanceScanBindData->connector->executeQuery(lanceScanBindData->query); - return std::make_unique(std::move(queryResult)); + auto bindData = input.bindData->constPtrCast(); + auto dataset = std::make_shared(bindData->datasetPath); + std::vector projection; + for (auto i = 0u; i < bindData->columnNames.size(); ++i) { + projection.push_back(LanceReader::ProjectedColumn{bindData->columnNames[i], + bindData->columnTypes[i].copy()}); + } + return std::make_unique(std::move(dataset), std::move(projection)); } offset_t tableFunc(const TableFuncInput& input, TableFuncOutput& output) { - auto sharedState = input.sharedState->ptrCast(); - auto lanceScanBindData = input.bindData->constPtrCast(); - std::unique_ptr result; - try { - // Duckdb queryResult.fetch() is not thread safe, we have to acquire a lock there. - std::lock_guard lock{sharedState->mtx}; - result = sharedState->queryResult->Fetch(); - } catch (std::exception& e) { + auto sharedState = input.sharedState->ptrCast(); + std::lock_guard lock{sharedState->mtx}; + auto projTypes = sharedState->reader.getProjectionTypes(); + if (projTypes.empty()) { return 0; } - if (result == nullptr) { + auto* context = input.context->clientContext; + auto memoryManager = storage::MemoryManager::Get(*context); + DataChunk slice(projTypes.size()); + for (auto i = 0u; i < projTypes.size(); ++i) { + slice.insert(i, std::make_shared(projTypes[i].copy(), memoryManager)); + } + slice.state->getSelVectorUnsafe().setToFiltered(0); + const auto rowsRead = sharedState->reader.next(slice); + if (rowsRead == 0) { return 0; } - lanceScanBindData->converter.convertDuckDBResultToVector(*result, output.dataChunk); + // Copy the slice into the output chunk (same types by construction). + for (auto i = 0u; i < projTypes.size(); ++i) { + auto& srcVector = slice.getValueVector(i); + auto& dstVector = output.dataChunk.getValueVectorMutable(i); + for (auto row = 0u; row < rowsRead; ++row) { + dstVector.copyFromVectorData(row, &srcVector, row); + } + } + output.dataChunk.state->getSelVectorUnsafe().setToUnfiltered(rowsRead); return output.dataChunk.state->getSelVector().getSelSize(); } diff --git a/lance/src/include/connector/lance_connector.h b/lance/src/include/connector/lance_connector.h deleted file mode 100644 index 6a716b03..00000000 --- a/lance/src/include/connector/lance_connector.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include "connector/duckdb_connector.h" - -namespace lbug { -namespace lance_extension { - -// DuckDB-backed connector for Lance datasets. Creates an in-memory DuckDB -// instance with the lance (and httpfs, for object-store URIs) extensions -// loaded, so Lance datasets can be queried with the same engine that powers -// lance-duckdb: `SELECT ... FROM '.lance'` (replacement scan), -// `COPY ... TO ... (FORMAT lance)`, and friends. -class LanceConnector : public duckdb_extension::DuckDBConnector { -public: - void connect(const std::string& dbPath, const std::string& catalogName, - const std::string& schemaName, main::ClientContext* context) override; - - // Opens an additional connection on the shared instance for one scan state. - // DuckDB connections must not run concurrent queries, so every scan state - // owns its connection and issues its range queries without locking. - std::unique_ptr newConnection(main::ClientContext* context); - - std::unique_ptr executeOn(duckdb::Connection& connection, - const std::string& query) const; - - // SQL builders. Identifiers are double-quoted, string literals single-quoted. - static std::string escapeLiteral(const std::string& value); - static std::string quoteIdent(const std::string& ident); - static std::string scanAllQuery(const std::string& datasetPath); - static std::string countRowsQuery(const std::string& datasetPath); - static std::string schemaQuery(const std::string& datasetPath); -}; - -} // namespace lance_extension -} // namespace lbug diff --git a/lance/src/include/function/lance_scan.h b/lance/src/include/function/lance_scan.h index 94c18642..42021a7f 100644 --- a/lance/src/include/function/lance_scan.h +++ b/lance/src/include/function/lance_scan.h @@ -1,20 +1,18 @@ #pragma once +#include + #include "binder/binder.h" -#include "connector/connector_factory.h" -#include "connector/duckdb_result_converter.h" -#include "connector/duckdb_type_converter.h" -#include "connector/lance_connector.h" #include "function/table/table_function.h" +#include "storage/lance_reader.h" namespace lbug { namespace lance_extension { -// Reads a Lance dataset through DuckDB's lance extension -// (lance-duckdb replacement scan: `SELECT * FROM '.lance'`). -// Registering LANCE_SCAN also enables `LOAD FROM ... (file_format='lance')` -// and `COPY ... FROM ... (file_format='lance')` via the `_SCAN` -// convention. +// Reads a Lance dataset through the native Lance reader (the same engine as +// lance-duckdb, without the DuckDB layer). Registering LANCE_SCAN also enables +// `LOAD FROM ... (file_format='lance')` and `COPY ... FROM ... +// (file_format='lance')` via the `_SCAN` convention. struct LanceScanFunction { static constexpr const char* name = "LANCE_SCAN"; @@ -22,24 +20,44 @@ struct LanceScanFunction { }; struct LanceScanBindData final : function::ScanFileBindData { - std::string query; - std::shared_ptr connector; - duckdb_extension::DuckDBResultConverter converter; + std::string datasetPath; + std::vector columnNames; + std::vector columnTypes; - LanceScanBindData(std::string query, - std::shared_ptr connector, - duckdb_extension::DuckDBResultConverter converter, binder::expression_vector columns, + LanceScanBindData(std::string datasetPath, std::vector columnNames, + std::vector columnTypes, binder::expression_vector columns, main::ClientContext* context) : function::ScanFileBindData{std::move(columns), 0 /* numRows */, common::FileScanInfo{}, context}, - query{std::move(query)}, connector{std::move(connector)}, - converter{std::move(converter)} {} + datasetPath{std::move(datasetPath)}, columnNames{std::move(columnNames)}, + columnTypes{std::move(columnTypes)} {} + + // LogicalType is not copyable; rebuild the type vector element-wise. + LanceScanBindData(const LanceScanBindData& other) + : function::ScanFileBindData{other}, datasetPath{other.datasetPath}, + columnNames{other.columnNames} { + for (auto& type : other.columnTypes) { + columnTypes.push_back(type.copy()); + } + } std::unique_ptr copy() const override { return std::make_unique(*this); } }; +struct LanceScanSharedState final : function::TableFuncSharedState { + explicit LanceScanSharedState(std::shared_ptr dataset, + std::vector projection) + : dataset{std::move(dataset)}, reader{this->dataset, std::move(projection)} { + reader.openRange(0, UINT64_MAX); + } + + std::mutex mtx; + std::shared_ptr dataset; + LanceReader reader; +}; + std::unique_ptr initLanceScanSharedState( const function::TableFuncInitSharedStateInput& input); diff --git a/lance/src/include/storage/lance_constants.h b/lance/src/include/storage/lance_constants.h index 04c1015f..e191568d 100644 --- a/lance/src/include/storage/lance_constants.h +++ b/lance/src/include/storage/lance_constants.h @@ -22,10 +22,9 @@ struct LanceConstants { static constexpr std::string_view DATASET_SUFFIX = ".lance"; - // Node-table scan morsel sizing. Morsels are row ranges read with - // LIMIT/OFFSET, which lance-duckdb pushes into the Lance scan, so each - // morsel is one efficient range query. Bounded to keep per-morsel - // materialized DuckDB results small. + // Node-table scan morsels are row ranges read with limit/offset pushed + // into the Lance scan, so each morsel is one efficient range query. + // Bounded to keep per-morsel materialized results small. static constexpr uint64_t MORSEL_TARGET_ROWS = 8192; static constexpr uint64_t MAX_MORSELS = 4096; }; diff --git a/lance/src/include/storage/lance_ffi.h b/lance/src/include/storage/lance_ffi.h new file mode 100644 index 00000000..5f29389a --- /dev/null +++ b/lance/src/include/storage/lance_ffi.h @@ -0,0 +1,128 @@ +#pragma once + +#include +#include +#include + +#include "common/arrow/arrow.h" +#include "common/exception/runtime.h" + +// C ABI of the lance_ladybug_ffi Rust staticlib (extension/lance/rust). +// Conventions mirror lance-duckdb's FFI: opaque void* handles, int return +// codes (0 ok, 1 exhausted where noted, -1 error), and thread-local error +// details via lance_last_error_* (taking the message transfers ownership to +// the caller, who frees it with lance_free_string). +extern "C" { + +void* lance_open_dataset(const char* path); +void lance_close_dataset(void* dataset); +int64_t lance_dataset_count_rows(void* dataset); +int lance_dataset_version(void* dataset, uint64_t* out_version); +int lance_schema_to_arrow(void* dataset, struct ArrowSchema* out_schema); +const char* lance_schema_version_string(void* dataset); +void* lance_scan_open(void* dataset, const char* const* columns, size_t columns_len, int64_t limit, + uint64_t offset); +int lance_scan_next(void* stream, void** out_batch); +void lance_close_scan(void* stream); +int lance_batch_to_arrow(void* batch, struct ArrowArray* out_array, struct ArrowSchema* out_schema); +void lance_free_batch(void* batch); +void lance_free_string(const char* string); +int lance_last_error_code(); +const char* lance_last_error_message(); + +} // extern "C" + +namespace lbug { +namespace lance_extension { + +// Throws RuntimeException with the pending Rust error (if any). +inline void checkLance(int rc, const std::string& what) { + if (rc == 0) { + return; + } + std::string detail; + if (const char* msg = lance_last_error_message()) { + detail = msg; + lance_free_string(msg); + } + throw common::RuntimeException{what + (detail.empty() ? "" : ": " + detail)}; +} + +struct LanceDatasetHandle { + void* handle = nullptr; + explicit LanceDatasetHandle(const std::string& path) { + handle = lance_open_dataset(path.c_str()); + if (!handle) { + checkLance(-1, "Failed to open Lance dataset '" + path + "'"); + } + } + ~LanceDatasetHandle() { + if (handle) { + lance_close_dataset(handle); + } + } + LanceDatasetHandle(const LanceDatasetHandle&) = delete; + LanceDatasetHandle& operator=(const LanceDatasetHandle&) = delete; + LanceDatasetHandle(LanceDatasetHandle&& other) noexcept : handle{other.handle} { + other.handle = nullptr; + } + LanceDatasetHandle& operator=(LanceDatasetHandle&& other) noexcept { + if (this != &other) { + if (handle) { + lance_close_dataset(handle); + } + handle = other.handle; + other.handle = nullptr; + } + return *this; + } +}; + +struct LanceScanHandle { + void* handle = nullptr; + LanceScanHandle() = default; + LanceScanHandle(const LanceScanHandle&) = delete; + LanceScanHandle& operator=(const LanceScanHandle&) = delete; + LanceScanHandle(LanceScanHandle&& other) noexcept : handle{other.handle} { + other.handle = nullptr; + } + LanceScanHandle& operator=(LanceScanHandle&& other) noexcept { + if (this != &other) { + close(); + handle = other.handle; + other.handle = nullptr; + } + return *this; + } + ~LanceScanHandle() { close(); } + void close() { + if (handle) { + lance_close_scan(handle); + handle = nullptr; + } + } +}; + +// One exported batch slice: the Rust batch is freed right after export (the +// exported array/schema hold their own references), so this only owns the +// Arrow C structs, released by the RAII wrappers on scope exit. +struct LanceBatchSlice { + ArrowArrayWrapper array; + ArrowSchemaWrapper schema; + LanceBatchSlice() = default; + LanceBatchSlice(const LanceBatchSlice&) = delete; + LanceBatchSlice& operator=(const LanceBatchSlice&) = delete; + LanceBatchSlice(LanceBatchSlice&&) = default; + LanceBatchSlice& operator=(LanceBatchSlice&&) = default; + + static LanceBatchSlice importBatch(void* rawBatch, const std::string& datasetPath) { + LanceBatchSlice slice; + checkLance(lance_batch_to_arrow(rawBatch, &slice.array, &slice.schema), + "Failed to export Lance batch from '" + datasetPath + "'"); + lance_free_batch(rawBatch); + return slice; + } +}; + +} // namespace lance_extension +} // namespace lbug diff --git a/lance/src/include/storage/lance_node_table.h b/lance/src/include/storage/lance_node_table.h index 353b1a4e..a7d956f7 100644 --- a/lance/src/include/storage/lance_node_table.h +++ b/lance/src/include/storage/lance_node_table.h @@ -19,8 +19,7 @@ namespace lance_extension { // `/nodes_
.lance`, or a single `.lance` dataset when storage // points at one directly. struct LanceNodeTableScanState final : storage::ColumnarNodeTableScanState { - std::shared_ptr connector; - std::string datasetPath; + std::shared_ptr dataset; std::unique_ptr reader; // Current morsel: global row range [morselStart, morselStart + morselLimit). uint64_t morselStart = 0; @@ -94,13 +93,13 @@ class LanceNodeTable final : public storage::ColumnarNodeTableBase { common::row_idx_t getTotalRowCount(const transaction::Transaction* transaction) const override; private: - std::shared_ptr connector; std::string datasetPath; - mutable std::mutex schemaMutex; + mutable std::mutex datasetMutex; + mutable std::shared_ptr dataset; mutable std::vector datasetSchema; mutable std::atomic cachedRowCount{common::INVALID_ROW_IDX}; - void ensureSchemaLoaded(main::ClientContext* context) const; + std::shared_ptr ensureDataset(main::ClientContext* context) const; void checkCompatibility(main::ClientContext* context) const; void initReaderForMorsel(transaction::Transaction* transaction, LanceNodeTableScanState& scanState) const; diff --git a/lance/src/include/storage/lance_reader.h b/lance/src/include/storage/lance_reader.h index db6843bf..8e922f7f 100644 --- a/lance/src/include/storage/lance_reader.h +++ b/lance/src/include/storage/lance_reader.h @@ -5,78 +5,78 @@ #include #include +#include "common/arrow/arrow_converter.h" #include "common/data_chunk/data_chunk.h" -#include "connector/duckdb_connector.h" -#include "connector/duckdb_result_converter.h" -#include "connector/duckdb_type_converter.h" -#include "connector/lance_connector.h" +#include "storage/lance_ffi.h" namespace lbug { namespace lance_extension { -// One dataset column: its name, its DuckDB type spelling, and the converted Ladybug type. +// One dataset column: its name and Ladybug type (mapped from the dataset's +// Arrow schema with ArrowConverter, so conversion of batch arrays with the +// same schema is exact by construction). struct LanceDatasetColumn { std::string name; - std::string duckdbType; common::LogicalType ladybugType; }; -// Stateful range reader over one Lance dataset. Issues one DuckDB/SQL query per -// range (`SELECT FROM '' LIMIT/OFFSET`, which lance-duckdb pushes -// into the Lance scan) and decodes result chunks into Ladybug DataChunks with -// DuckDBResultConverter. One reader serves one scan state; ranges are opened -// sequentially (a new range replaces the previous query), so no locking is -// needed. Mirrors the ParquetReader surface used by the icebug-disk tables. +// Shared, thread-safe handle to one open Lance dataset (a stable snapshot of +// the latest version at open time). Tables hold one; scan states borrow it +// for their range scans. +class LanceDataset { +public: + explicit LanceDataset(std::string datasetPath); + ~LanceDataset() = default; + LanceDataset(const LanceDataset&) = delete; + LanceDataset& operator=(const LanceDataset&) = delete; + + const std::string& getPath() const { return datasetPath; } + uint64_t countRows(); + uint64_t version(); + // Dataset schema: column names and mapped types, in dataset order. + std::vector schema(); + // The `lance_version` schema-metadata marker, or empty when absent. + std::string versionString(); + + void* handle() { return dataset.handle; } + +private: + std::string datasetPath; + LanceDatasetHandle dataset; + std::mutex mutex; + bool rowCountCached = false; + uint64_t cachedRowCount = 0; +}; + +// Stateful range reader over one Lance dataset. Opens one scan per range +// (`SELECT`-equivalent projection + limit/offset pushed into Lance) and +// decodes slices into Ladybug DataChunks with ArrowConverter. One reader +// serves one scan state; ranges open sequentially, so no locking is needed. class LanceReader { public: - // One projected output column: the dataset column to read, the DuckDB CAST - // target aligning it with the catalog type, and the Ladybug output type. + // One projected output column: the dataset column name and Ladybug type. + // Types must match the dataset schema exactly (validated at CREATE). struct ProjectedColumn { std::string datasetName; - std::string castSpelling; common::LogicalType ladybugType; }; - LanceReader(std::shared_ptr connector, std::string datasetPath, - std::vector projection, main::ClientContext* context); + LanceReader(std::shared_ptr dataset, std::vector projection); - const std::vector& getProjection() const { return projection; } - std::vector getProjectionTypes() const { - std::vector types; - types.reserve(projection.size()); - for (auto& col : projection) { - types.push_back(col.ladybugType.copy()); - } - return types; - } - - // Issues a new range query, replacing any active one. limit == UINT64_MAX scans to end. + // Issues a new range scan, replacing any active one. limit == UINT64_MAX scans to end. void openRange(uint64_t offset, uint64_t limit); - // Fetches the next chunk of the active range into a fresh `out` (whose vectors must + // Converts the next slice of the active range into a fresh `out` (whose vectors must // match the projection in order). Returns rows read; 0 means exhausted. uint64_t next(common::DataChunk& out); - // Metadata helpers (each opens a short-lived connection). - static std::vector fetchSchema(LanceConnector& connector, - main::ClientContext* context, const std::string& datasetPath); - static uint64_t countRows(LanceConnector& connector, main::ClientContext* context, - const std::string& datasetPath); - - // DuckDB CAST target spelling for a catalog type, so decoded values have the physical - // layout DuckDBResultConverter expects. Throws for STRUCT/MAP/UNION and other types - // with no lossless spelling (see docs/lance.md, supported types). - static std::string toDuckDBCastSpelling(const common::LogicalType& type); + const std::vector& getProjection() const { return projection; } + std::vector getProjectionTypes() const; private: - std::string buildRangeQuery(uint64_t offset, uint64_t limit) const; - - std::shared_ptr connector; - std::string datasetPath; + std::shared_ptr dataset; std::vector projection; - main::ClientContext* context; - std::unique_ptr connection; - std::unique_ptr activeResult; - duckdb_extension::DuckDBResultConverter converter; + LanceScanHandle scan; + std::string datasetPath; }; } // namespace lance_extension diff --git a/lance/src/include/storage/lance_rel_table.h b/lance/src/include/storage/lance_rel_table.h index 2ceffe8a..6b0a5ac5 100644 --- a/lance/src/include/storage/lance_rel_table.h +++ b/lance/src/include/storage/lance_rel_table.h @@ -22,6 +22,7 @@ enum class LanceRelTableLayout : uint8_t { CSR, FLAT }; // multi-rel scans; see ExternalTableHooks::scanStateRank. The Parquet reader members // inherited from the icebug-disk state stay null; Lance reads go through `reader`. struct LanceRelTableScanState final : storage::IceDiskRelTableScanState { + std::shared_ptr dataset; std::unique_ptr reader; // Maps scan-output positions to cached-batch positions (catalog column ID order is // not assumed to match dataset order; edge properties match by name). @@ -70,18 +71,18 @@ class LanceRelTable final : public storage::ColumnarRelTableBase { common::idx_t k) const override; private: - std::shared_ptr connector; LanceRelTableLayout layout; std::string indicesDatasetPath; std::string indptrDatasetPath; - mutable std::mutex schemaMutex; + mutable std::mutex datasetMutex; + mutable std::shared_ptr dataset; mutable std::vector indicesSchema; mutable std::mutex indptrDataMutex; mutable std::vector indptrData; mutable std::atomic indptrDataLoaded{false}; mutable std::atomic cachedRowCount{common::INVALID_ROW_IDX}; - void ensureSchemasLoaded(main::ClientContext* context) const; + std::shared_ptr ensureDataset(main::ClientContext* context) const; void checkCompatibility(main::ClientContext* context) const; void loadIndptrData(transaction::Transaction* transaction) const; void ensureReader(transaction::Transaction* transaction, diff --git a/lance/src/installer/CMakeLists.txt b/lance/src/installer/CMakeLists.txt deleted file mode 100644 index de2df4e5..00000000 --- a/lance/src/installer/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -if (NOT WIN32) - add_library(lance_installer - SHARED - lance_install_func.cpp - ${PROJECT_SOURCE_DIR}/extension/duckdb/src/installer/duckdb_installer.cpp) - set_extension_properties(lance_installer lance_installer lance) -endif () - -if (APPLE) - set_apple_dynamic_lookup(lance_installer) -endif () diff --git a/lance/src/installer/lance_install_func.cpp b/lance/src/installer/lance_install_func.cpp deleted file mode 100644 index 4315d417..00000000 --- a/lance/src/installer/lance_install_func.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "installer/duckdb_installer.h" - -extern "C" { -// Because we link against the static library on windows, we implicitly inherit LBUG_STATIC_DEFINE, -// which cancels out any exporting, so we can't use LBUG_API. -#if defined(_WIN32) -#define INIT_EXPORT __declspec(dllexport) -#else -#define INIT_EXPORT __attribute__((visibility("default"))) -#endif -INIT_EXPORT void install(const std::string& repo, lbug::main::ClientContext& context) { - lbug::extension::InstallExtensionInfo info{"lance", repo, false /* forceInstall */}; - lbug::duckdb_extension::DuckDBInstaller installer{info, context}; - installer.install(); -} -} diff --git a/lance/src/main/CMakeLists.txt b/lance/src/main/CMakeLists.txt index c0e7a7f2..984fc08b 100644 --- a/lance/src/main/CMakeLists.txt +++ b/lance/src/main/CMakeLists.txt @@ -1,8 +1,6 @@ add_library(lance_extension_main OBJECT - lance_extension.cpp - ${PROJECT_SOURCE_DIR}/extension/duckdb/src/main/duckdb_options.cpp - ${PROJECT_SOURCE_DIR}/extension/httpfs/src/s3fs_config.cpp) + lance_extension.cpp) set(LANCE_EXTENSION_OBJECT_FILES ${LANCE_EXTENSION_OBJECT_FILES} $ diff --git a/lance/src/main/lance_extension.cpp b/lance/src/main/lance_extension.cpp index ea031beb..0c91ce9a 100644 --- a/lance/src/main/lance_extension.cpp +++ b/lance/src/main/lance_extension.cpp @@ -3,7 +3,6 @@ #include "function/lance_scan.h" #include "main/client_context.h" #include "main/database.h" -#include "main/duckdb_extension.h" #include "storage/lance_node_table.h" #include "storage/lance_rel_table.h" #include "storage/table/external_table_hooks.h" @@ -14,7 +13,6 @@ namespace lance_extension { void LanceExtension::load(main::ClientContext* context) { auto& db = *context->getDatabase(); extension::ExtensionUtils::addTableFunc(db); - duckdb_extension::DuckdbExtension::loadRemoteFSOptions(context); // Register Lance-backed table storage for StorageFormat::LANCE. The scan-state // rank (3) declares that Lance scan states extend the icebug-disk scan state and // therefore cover every lower-rank backend in shared multi-rel scans. diff --git a/lance/src/storage/lance_node_table.cpp b/lance/src/storage/lance_node_table.cpp index afc640a6..64b66035 100644 --- a/lance/src/storage/lance_node_table.cpp +++ b/lance/src/storage/lance_node_table.cpp @@ -1,6 +1,7 @@ #include "storage/lance_node_table.h" #include +#include #include #include "catalog/catalog_entry/node_table_catalog_entry.h" @@ -30,64 +31,72 @@ LanceNodeTable::LanceNodeTable(const StorageManager* storageManager, const NodeTableCatalogEntry* nodeTableEntry, MemoryManager* memoryManager, main::ClientContext* context) : ColumnarNodeTableBase{storageManager, nodeTableEntry, memoryManager, - std::make_unique()}, - connector{std::make_shared()} { - connector->connect("" /* inMemDB */, "" /* defaultCatalogName */, "" /* defaultSchemaName */, - context); + std::make_unique()} { const auto& storage = nodeTableEntry->getStorage(); datasetPath = common::VirtualFileSystem::resolvePath(context, LanceUtils::constructNodeTablePath(storage, nodeTableEntry->getName())); // Object-store datasets are validated lazily at scan time (credentials may only be // configured later); local datasets fail fast here, mirroring icebug-disk. if (common::LocalFileSystem::isLocalPath(datasetPath)) { - ensureSchemaLoaded(context); + ensureDataset(context); checkCompatibility(context); } } -void LanceNodeTable::ensureSchemaLoaded(main::ClientContext* context) const { - if (!datasetSchema.empty()) { - return; +std::shared_ptr LanceNodeTable::ensureDataset(main::ClientContext* context) const { + (void)context; + if (dataset) { + return dataset; } - std::lock_guard lock{schemaMutex}; - if (!datasetSchema.empty()) { - return; + std::lock_guard lock{datasetMutex}; + if (!dataset) { + dataset = std::make_shared(datasetPath); + datasetSchema = dataset->schema(); } - datasetSchema = LanceReader::fetchSchema(*connector, context, datasetPath); + return dataset; } void LanceNodeTable::checkCompatibility(main::ClientContext* context) const { + // Enforce the stamped schema version. Datasets without the marker predate + // versioning and keep working with a warning, mirroring icebug-disk. + const auto version = dataset->versionString(); + if (version.empty()) { + std::cerr << "Warning: " << datasetPath + << ": Lance dataset is missing lance_version schema metadata; continuing " + "without version compatibility check" + << std::endl; + } else if (version != LanceConstants::CURRENT_VERSION) { + throw RuntimeException{datasetPath + ": current ladybug version does not support " + + "lance_version: " + version}; + } // The primary key column must exist: node offsets are positional, but every filter // and lookup addresses nodes through the PK. const auto pkName = nodeTableCatalogEntry->getPrimaryKeyName(); - const bool hasPK = std::any_of(datasetSchema.begin(), datasetSchema.end(), + const auto pkIt = std::find_if(datasetSchema.begin(), datasetSchema.end(), [&](const LanceDatasetColumn& col) { return col.name == pkName; }); - if (!hasPK) { + if (pkIt == datasetSchema.end()) { throw RuntimeException{"Lance dataset '" + datasetPath + "' has no primary key column '" + pkName + "' for node table '" + nodeTableCatalogEntry->getName() + "'"}; } - // Every catalog property present in the dataset must have a mappable type, and the - // full projection must bind (trial query over zero rows validates the CASTs). - std::string selectList; + // Declared properties present in the dataset must match exactly: values cross + // the FFI with dataset types, so any coercion would silently mistype them. + // Missing properties read as NULL, mirroring icebug-disk. for (auto i = 0u; i < nodeTableCatalogEntry->getNumProperties(); ++i) { const auto& prop = nodeTableCatalogEntry->getProperty(i); auto it = std::find_if(datasetSchema.begin(), datasetSchema.end(), [&](const LanceDatasetColumn& col) { return col.name == prop.getName(); }); if (it == datasetSchema.end()) { - continue; // Missing properties read as NULL, mirroring icebug-disk. + continue; } - const auto spelling = LanceReader::toDuckDBCastSpelling(prop.getType()); - if (!selectList.empty()) { - selectList += ", "; + if (it->ladybugType != prop.getType()) { + throw RuntimeException{"Lance dataset '" + datasetPath + "' column '" + it->name + + "' has type " + it->ladybugType.toString() + + " but node table '" + nodeTableCatalogEntry->getName() + + "' declares " + prop.getType().toString()}; } - selectList += "CAST(" + LanceConnector::quoteIdent(it->name) + " AS " + spelling + ")"; - } - if (!selectList.empty()) { - auto conn = connector->newConnection(context); - connector->executeOn(*conn, "SELECT " + selectList + " FROM '" + - LanceConnector::escapeLiteral(datasetPath) + "' LIMIT 0"); } + (void)context; } void LanceNodeTable::initializeScanCoordination(const transaction::Transaction* transaction) { @@ -115,23 +124,20 @@ void LanceNodeTable::initReaderForMorsel(Transaction* transaction, if (!context) { throw RuntimeException("Invalid client context for Lance node scan initialization"); } - ensureSchemaLoaded(context); + auto lanceDataset = ensureDataset(context); if (!scanState.reader) { - // First init for this scan state: snapshot the requested columns and build the - // reader projection (dataset column + CAST spelling + catalog type per column). + // First init for this scan state: snapshot the requested columns. Columns are + // matched by name; dataset types were validated against the catalog at CREATE. std::vector projection; std::vector readerColumnIDs; for (auto columnID : scanState.columnIDs) { if (columnID == INVALID_COLUMN_ID || columnID == ROW_IDX_COLUMN_ID) { continue; } - // Find the catalog property behind this column ID. std::string propName; - const LogicalType* propType = nullptr; for (auto i = 0u; i < nodeTableCatalogEntry->getNumProperties(); ++i) { if (nodeTableCatalogEntry->getColumnID(i) == columnID) { propName = nodeTableCatalogEntry->getProperty(i).getName(); - propType = &nodeTableCatalogEntry->getProperty(i).getType(); break; } } @@ -143,8 +149,7 @@ void LanceNodeTable::initReaderForMorsel(Transaction* transaction, if (it == datasetSchema.end()) { continue; // Filled with NULL below, mirroring icebug-disk. } - projection.push_back(LanceReader::ProjectedColumn{it->name, - LanceReader::toDuckDBCastSpelling(*propType), propType->copy()}); + projection.push_back(LanceReader::ProjectedColumn{it->name, it->ladybugType.copy()}); readerColumnIDs.push_back(columnID); } if (projection.empty()) { @@ -154,11 +159,11 @@ void LanceNodeTable::initReaderForMorsel(Transaction* transaction, throw RuntimeException{"Lance dataset '" + datasetPath + "' has no columns"}; } projection.push_back(LanceReader::ProjectedColumn{datasetSchema[0].name, - LanceReader::toDuckDBCastSpelling(datasetSchema[0].ladybugType), datasetSchema[0].ladybugType.copy()}); } + scanState.dataset = lanceDataset; scanState.reader = - std::make_unique(connector, datasetPath, std::move(projection), context); + std::make_unique(std::move(lanceDataset), std::move(projection)); scanState.readerColumnIDs = std::move(readerColumnIDs); } // Pull the next morsel from the shared state (mirrors IceDiskNodeTable: the @@ -199,8 +204,8 @@ bool LanceNodeTable::scanInternal(Transaction* transaction, TableScanState& scan auto context = transaction->getClientContext(); auto memoryManager = MemoryManager::Get(*context); - // Dataset column index per requested output position (INVALID_COLUMN_ID = NULL fill). - // Surrogate projection (no columns requested): single temp column, values discarded. + // Dataset column index per requested output position (surrogate projection when no + // columns were requested: single temp column, values discarded). const bool surrogate = lanceScanState.readerColumnIDs.empty(); const auto numProjCols = surrogate ? 1 : static_cast(lanceScanState.readerColumnIDs.size()); @@ -208,7 +213,6 @@ bool LanceNodeTable::scanInternal(Transaction* transaction, TableScanState& scan while (true) { scanState.resetOutVectors(); DataChunk lanceDataChunk(numProjCols, scanState.outState); - // NOTE: projection column types are snapshotted from the reader below. auto projTypes = lanceScanState.reader->getProjectionTypes(); for (auto i = 0u; i < numProjCols; ++i) { lanceDataChunk.insert(i, @@ -289,8 +293,7 @@ row_idx_t LanceNodeTable::getTotalRowCount(const Transaction* transaction) const return 0; } try { - const auto count = - static_cast(LanceReader::countRows(*connector, context, datasetPath)); + const auto count = static_cast(ensureDataset(context)->countRows()); cachedRowCount.store(count, std::memory_order_relaxed); return count; } catch (const std::exception&) { @@ -310,7 +313,9 @@ bool LanceNodeTable::isVisibleNoLock(const transaction::Transaction* transaction bool LanceNodeTable::lookupPK(const transaction::Transaction* transaction, common::ValueVector* keyVector, uint64_t vectorPos, common::offset_t& result) const { - // Same fast path as IceDiskNodeTable: INTERNAL_ID keys resolve directly to offsets. + // Lookup joins resolve INTERNAL_ID keys directly to offsets (same fast path as + // NodeTable::lookupPK). The base implementation checks in-memory node groups, + // which are empty for disk-resident tables, so handle it here via total row count. if (keyVector->dataType.getLogicalTypeID() == common::LogicalTypeID::INTERNAL_ID) { if (keyVector->isNull(vectorPos)) { return false; diff --git a/lance/src/storage/lance_reader.cpp b/lance/src/storage/lance_reader.cpp index 5a618d66..dbfc9b68 100644 --- a/lance/src/storage/lance_reader.cpp +++ b/lance/src/storage/lance_reader.cpp @@ -1,6 +1,8 @@ #include "storage/lance_reader.h" -#include "common/exception/binder.h" +#include +#include + #include "common/exception/runtime.h" #include @@ -9,167 +11,131 @@ namespace lance_extension { using namespace common; -std::string LanceReader::toDuckDBCastSpelling(const LogicalType& type) { - switch (type.getLogicalTypeID()) { - case LogicalTypeID::BOOL: - return "BOOLEAN"; - case LogicalTypeID::INT8: - return "TINYINT"; - case LogicalTypeID::INT16: - return "SMALLINT"; - case LogicalTypeID::INT32: - return "INTEGER"; - case LogicalTypeID::INT64: - return "BIGINT"; - case LogicalTypeID::UINT8: - return "UTINYINT"; - case LogicalTypeID::UINT16: - return "USMALLINT"; - case LogicalTypeID::UINT32: - return "UINTEGER"; - case LogicalTypeID::UINT64: - return "UBIGINT"; - case LogicalTypeID::INT128: - return "HUGEINT"; - case LogicalTypeID::UINT128: - return "UHUGEINT"; - case LogicalTypeID::FLOAT: - return "FLOAT"; - case LogicalTypeID::DOUBLE: - return "DOUBLE"; - case LogicalTypeID::STRING: - return "VARCHAR"; - case LogicalTypeID::BLOB: - return "BLOB"; - case LogicalTypeID::DATE: - return "DATE"; - case LogicalTypeID::TIMESTAMP: - return "TIMESTAMP"; - case LogicalTypeID::TIMESTAMP_SEC: - return "TIMESTAMP_S"; - case LogicalTypeID::TIMESTAMP_MS: - return "TIMESTAMP_MS"; - case LogicalTypeID::TIMESTAMP_NS: - return "TIMESTAMP_NS"; - case LogicalTypeID::TIMESTAMP_TZ: - return "TIMESTAMPTZ"; - case LogicalTypeID::INTERVAL: - return "INTERVAL"; - case LogicalTypeID::UUID: - return "UUID"; - case LogicalTypeID::DECIMAL: - return std::format("DECIMAL({},{})", DecimalType::getPrecision(type), - DecimalType::getScale(type)); - case LogicalTypeID::LIST: - return toDuckDBCastSpelling(ListType::getChildType(type)) + "[]"; - case LogicalTypeID::ARRAY: - return std::format("{}[{}]", toDuckDBCastSpelling(ArrayType::getChildType(type)), - ArrayType::getNumElements(type)); - case LogicalTypeID::STRUCT: - case LogicalTypeID::MAP: - case LogicalTypeID::UNION: - throw BinderException{std::format( - "Lance-backed tables do not support {} columns in v1. Flatten nested objects into " - "top-level columns (see docs/lance.md).", - LogicalTypeUtils::toString(type.getLogicalTypeID()))}; - default: - throw BinderException{std::format( - "Lance-backed tables do not support column type {} in v1.", type.toString())}; +LanceDataset::LanceDataset(std::string datasetPath_) + : datasetPath{std::move(datasetPath_)}, dataset{datasetPath} {} + +uint64_t LanceDataset::countRows() { + std::lock_guard lock{mutex}; + if (rowCountCached) { + return cachedRowCount; + } + const auto rows = lance_dataset_count_rows(dataset.handle); + if (rows < 0) { + checkLance(-1, "Failed to count rows of Lance dataset '" + datasetPath + "'"); } + cachedRowCount = static_cast(rows); + rowCountCached = true; + return cachedRowCount; } -LanceReader::LanceReader(std::shared_ptr connector_, std::string datasetPath_, - std::vector projection_, main::ClientContext* context_) - : connector{std::move(connector_)}, datasetPath{std::move(datasetPath_)}, - projection{std::move(projection_)}, context{context_}, converter{[this] { - std::vector types; - types.reserve(projection.size()); - for (auto& col : projection) { - types.push_back(col.ladybugType.copy()); - } - return types; - }()} {} +uint64_t LanceDataset::version() { + uint64_t out = 0; + checkLance(lance_dataset_version(dataset.handle, &out), + "Failed to read version of Lance dataset '" + datasetPath + "'"); + return out; +} -std::string LanceReader::buildRangeQuery(uint64_t offset, uint64_t limit) const { - std::string selectList; - if (projection.empty()) { - // No property columns requested (e.g. COUNT over node IDs, which are positional): - // read one arbitrary column and discard the values; only the row count matters. - // The caller substitutes a surrogate projection (see LanceNodeTable::scanInternal). - throw RuntimeException{"LanceReader requires a non-empty projection"}; - } - for (auto i = 0u; i < projection.size(); ++i) { - if (i > 0) { - selectList += ", "; - } - selectList += std::format("CAST({} AS {})", - LanceConnector::quoteIdent(projection[i].datasetName), projection[i].castSpelling); +std::vector LanceDataset::schema() { + ArrowSchemaWrapper schema; + checkLance(lance_schema_to_arrow(dataset.handle, &schema), + "Failed to read schema of Lance dataset '" + datasetPath + "'"); + std::vector columns; + for (int64_t i = 0; i < schema.n_children; ++i) { + const auto* child = schema.children[i]; + columns.push_back(LanceDatasetColumn{child->name ? child->name : "", + ArrowConverter::fromArrowSchema(child)}); } - auto query = - std::format("SELECT {} FROM '{}'", selectList, LanceConnector::escapeLiteral(datasetPath)); - if (offset > 0) { - query += std::format(" OFFSET {}", offset); + return columns; +} + +std::string LanceDataset::versionString() { + const char* raw = lance_schema_version_string(dataset.handle); + if (!raw) { + checkLance(-1, "Failed to read schema metadata of Lance dataset '" + datasetPath + "'"); } - if (limit != UINT64_MAX) { - query += std::format(" LIMIT {}", limit); + std::string out = raw; + lance_free_string(raw); + return out; +} + +LanceReader::LanceReader(std::shared_ptr dataset_, + std::vector projection_) + : dataset{std::move(dataset_)}, projection{std::move(projection_)}, + datasetPath{dataset->getPath()} {} + +std::vector LanceReader::getProjectionTypes() const { + std::vector types; + types.reserve(projection.size()); + for (auto& col : projection) { + types.push_back(col.ladybugType.copy()); } - return query; + return types; } void LanceReader::openRange(uint64_t offset, uint64_t limit) { - if (!connection) { - connection = connector->newConnection(context); + scan.close(); + std::vector columns; + columns.reserve(projection.size()); + for (auto& col : projection) { + columns.push_back(col.datasetName.c_str()); } - activeResult = connector->executeOn(*connection, buildRangeQuery(offset, limit)); + const int64_t ffiLimit = + limit == UINT64_MAX ? -1 : static_cast(std::min(limit, INT64_MAX)); + void* stream = + lance_scan_open(dataset->handle(), columns.data(), columns.size(), ffiLimit, offset); + if (!stream) { + checkLance(-1, "Failed to open Lance scan on '" + datasetPath + "' at offset " + + std::to_string(offset)); + } + scan.handle = stream; } uint64_t LanceReader::next(DataChunk& out) { - if (!activeResult) { + if (!scan.handle) { return 0; } - std::unique_ptr chunk; - try { - chunk = activeResult->Fetch(); - } catch (std::exception& e) { - throw RuntimeException{ - std::string{"Lance scan failed for '"} + datasetPath + "': " + e.what()}; - } - if (!chunk || chunk->size() == 0) { - activeResult.reset(); + void* rawBatch = nullptr; + const int rc = lance_scan_next(scan.handle, &rawBatch); + if (rc == 1) { + scan.close(); return 0; } - converter.convertDuckDBResultToVector(*chunk, out); - return out.state->getSelVector().getSelSize(); -} - -std::vector LanceReader::fetchSchema(LanceConnector& connector, - main::ClientContext* context, const std::string& datasetPath) { - auto conn = connector.newConnection(context); - auto result = connector.executeOn(*conn, LanceConnector::schemaQuery(datasetPath)); - const auto& types = result->GetTypes(); - const auto& names = result->GetNames(); - if (types.size() != names.size()) { - throw RuntimeException{"Failed to read Lance dataset schema for '" + datasetPath + "'"}; + checkLance(rc, "Failed to read Lance dataset '" + datasetPath + "'"); + auto slice = LanceBatchSlice::importBatch(rawBatch, datasetPath); + const auto rows = static_cast(slice.array.length); + // Detach vector states for conversion: ArrowConverter consults the output + // vector state, and the caller's (usually filtered) chunk state would + // suppress all writes. Restored before returning. + std::vector> savedStates; + for (auto i = 0u; i < projection.size(); ++i) { + auto& vec = out.getValueVectorMutable(i); + savedStates.push_back(vec.state); + vec.setState(nullptr); } - std::vector schema; - for (auto i = 0u; i < types.size(); ++i) { - const auto typeStr = types[i].ToString(); - schema.push_back(LanceDatasetColumn{names[i].GetIdentifierName(), typeStr, - duckdb_extension::DuckDBTypeConverter::convertDuckDBType(typeStr)}); + // Match batch columns to the projection by name (scan output order is not + // part of the contract). + for (auto i = 0u; i < projection.size(); ++i) { + const auto* childSchema = static_cast(nullptr); + const auto* childArray = static_cast(nullptr); + for (int64_t j = 0; j < slice.schema.n_children; ++j) { + const char* name = slice.schema.children[j]->name; + if (name && projection[i].datasetName == name) { + childSchema = slice.schema.children[j]; + childArray = slice.array.children[j]; + break; + } + } + if (!childSchema || !childArray) { + throw RuntimeException{"Lance scan of '" + datasetPath + "' is missing projected " + + "column '" + projection[i].datasetName + "'"}; + } + ArrowConverter::fromArrowArray(childSchema, childArray, out.getValueVectorMutable(i)); } - return schema; -} - -uint64_t LanceReader::countRows(LanceConnector& connector, main::ClientContext* context, - const std::string& datasetPath) { - auto conn = connector.newConnection(context); - auto result = connector.executeOn(*conn, LanceConnector::countRowsQuery(datasetPath)); - auto chunk = result->Fetch(); - if (!chunk || chunk->size() == 0) { - throw RuntimeException{"Failed to count rows of Lance dataset '" + datasetPath + "'"}; + for (auto i = 0u; i < projection.size(); ++i) { + out.getValueVectorMutable(i).setState(savedStates[i]); } - // COUNT(*) returns BIGINT. - return static_cast(chunk->GetValue(0, 0).GetValue()); + out.state->getSelVectorUnsafe().setToUnfiltered(rows); + return rows; } } // namespace lance_extension diff --git a/lance/src/storage/lance_rel_table.cpp b/lance/src/storage/lance_rel_table.cpp index d3f033c9..c520db00 100644 --- a/lance/src/storage/lance_rel_table.cpp +++ b/lance/src/storage/lance_rel_table.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -35,9 +36,7 @@ LanceRelTable::LanceRelTable(RelGroupCatalogEntry* relGroupEntry, table_id_t fro table_id_t toTableID, const StorageManager* storageManager, MemoryManager* memoryManager, main::ClientContext* context) : ColumnarRelTableBase{relGroupEntry, fromTableID, toTableID, storageManager, memoryManager}, - connector{std::make_shared()}, layout{LanceRelTableLayout::CSR} { - connector->connect("" /* inMemDB */, "" /* defaultCatalogName */, "" /* defaultSchemaName */, - context); + layout{LanceRelTableLayout::CSR} { const auto& storage = relGroupEntry->getStorage(); if (LanceUtils::isSingleDatasetPath(storage)) { layout = LanceRelTableLayout::FLAT; @@ -50,53 +49,74 @@ LanceRelTable::LanceRelTable(RelGroupCatalogEntry* relGroupEntry, table_id_t fro // Object-store datasets are validated lazily at scan time; local datasets fail fast // here, mirroring icebug-disk. if (common::LocalFileSystem::isLocalPath(indicesDatasetPath)) { - ensureSchemasLoaded(context); + ensureDataset(context); checkCompatibility(context); } } -void LanceRelTable::ensureSchemasLoaded(main::ClientContext* context) const { - if (!indicesSchema.empty()) { - return; +std::shared_ptr LanceRelTable::ensureDataset(main::ClientContext* context) const { + (void)context; + if (dataset) { + return dataset; } - std::lock_guard lock{schemaMutex}; - if (!indicesSchema.empty()) { - return; + std::lock_guard lock{datasetMutex}; + if (!dataset) { + dataset = std::make_shared(indicesDatasetPath); + indicesSchema = dataset->schema(); + } + return dataset; +} + +static void requireExactType(const std::string& datasetPath, const std::string& tableName, + const std::string& columnName, const LogicalType& datasetType, const LogicalType& catalogType) { + if (datasetType != catalogType) { + throw RuntimeException{"Lance dataset '" + datasetPath + "' column '" + columnName + + "' has type " + datasetType.toString() + " but table '" + tableName + + "' declares " + catalogType.toString()}; } - indicesSchema = LanceReader::fetchSchema(*connector, context, indicesDatasetPath); } void LanceRelTable::checkCompatibility(main::ClientContext* context) const { + const auto version = dataset->versionString(); + if (version.empty()) { + std::cerr << "Warning: " << indicesDatasetPath + << ": Lance dataset is missing lance_version schema metadata; continuing " + "without version compatibility check" + << std::endl; + } else if (version != LanceConstants::CURRENT_VERSION) { + throw RuntimeException{ + indicesDatasetPath + + ": current ladybug version does not support lance_version: " + version}; + } if (indicesSchema.empty()) { throw RuntimeException{"Lance dataset '" + indicesDatasetPath + "' has no columns"}; } - std::vector trial; + const auto& tableName = relGroupEntry->getName(); if (layout == LanceRelTableLayout::FLAT) { if (indicesSchema.size() < 2) { throw RuntimeException{"Flat Lance rel dataset '" + indicesDatasetPath + "' requires source and target offset columns"}; } - trial.push_back( - LanceReader::ProjectedColumn{indicesSchema[0].name, "BIGINT", LogicalType::INT64()}); - trial.push_back( - LanceReader::ProjectedColumn{indicesSchema[1].name, "BIGINT", LogicalType::INT64()}); + requireExactType(indicesDatasetPath, tableName, indicesSchema[0].name, + indicesSchema[0].ladybugType, LogicalType::INT64()); + requireExactType(indicesDatasetPath, tableName, indicesSchema[1].name, + indicesSchema[1].ladybugType, LogicalType::INT64()); } else { - // Column 0 is the destination node offset. - trial.push_back( - LanceReader::ProjectedColumn{indicesSchema[0].name, "BIGINT", LogicalType::INT64()}); - auto indptrSchema = LanceReader::fetchSchema(*connector, context, indptrDatasetPath); + // Column 0 is the destination node offset; the indptr dataset holds one + // integer column with one entry per source node plus a sentinel. + requireExactType(indicesDatasetPath, tableName, indicesSchema[0].name, + indicesSchema[0].ladybugType, LogicalType::INT64()); + auto indptrDataset = std::make_shared(indptrDatasetPath); + const auto indptrSchema = indptrDataset->schema(); if (indptrSchema.empty()) { throw RuntimeException{ "Lance indptr dataset '" + indptrDatasetPath + "' has no columns"}; } - auto conn = connector->newConnection(context); - connector->executeOn(*conn, - "SELECT CAST(" + LanceConnector::quoteIdent(indptrSchema[0].name) + - " AS BIGINT) FROM '" + LanceConnector::escapeLiteral(indptrDatasetPath) + - "' LIMIT 0"); - } - // Edge properties match by name; every present property must have a mappable type. - // Missing properties read as NULL, mirroring icebug-disk. + requireExactType(indptrDatasetPath, tableName, indptrSchema[0].name, + indptrSchema[0].ladybugType, LogicalType::INT64()); + } + // Edge properties match by name and must match exactly; missing properties + // read as NULL. for (auto i = 0u; i < relGroupEntry->getNumProperties(); ++i) { const auto& prop = relGroupEntry->getProperty(i); if (prop.getType().getLogicalTypeID() == LogicalTypeID::INTERNAL_ID) { @@ -107,22 +127,9 @@ void LanceRelTable::checkCompatibility(main::ClientContext* context) const { if (it == indicesSchema.end()) { continue; } - trial.push_back(LanceReader::ProjectedColumn{it->name, - LanceReader::toDuckDBCastSpelling(prop.getType()), prop.getType().copy()}); - } - // Trial query over zero rows validates that every CAST binds. - std::string selectList; - for (auto& col : trial) { - if (!selectList.empty()) { - selectList += ", "; - } - selectList += - "CAST(" + LanceConnector::quoteIdent(col.datasetName) + " AS " + col.castSpelling + ")"; + requireExactType(indicesDatasetPath, tableName, it->name, it->ladybugType, prop.getType()); } - auto conn = connector->newConnection(context); - connector->executeOn(*conn, "SELECT " + selectList + " FROM '" + - LanceConnector::escapeLiteral(indicesDatasetPath) + - "' LIMIT 0"); + (void)context; } void LanceRelTable::ensureReader(Transaction* transaction, @@ -134,13 +141,10 @@ void LanceRelTable::ensureReader(Transaction* transaction, if (!context) { throw RuntimeException("Invalid client context for Lance rel scan initialization"); } - ensureSchemasLoaded(context); + auto lanceDataset = ensureDataset(context); // Projection: endpoint offset column(s) first, then edge properties present in the // dataset (by name), in catalog column order. outputToCached maps each requested // output position to its cached-batch position (-1 = NULL fill). - if (indicesSchema.empty()) { - throw RuntimeException{"Lance dataset '" + indicesDatasetPath + "' has no columns"}; - } std::vector projection; std::vector projColumnIDs; if (layout == LanceRelTableLayout::FLAT) { @@ -149,14 +153,14 @@ void LanceRelTable::ensureReader(Transaction* transaction, "' requires source and target offset columns"}; } projection.push_back( - LanceReader::ProjectedColumn{indicesSchema[0].name, "BIGINT", LogicalType::INT64()}); + LanceReader::ProjectedColumn{indicesSchema[0].name, LogicalType::INT64()}); projColumnIDs.push_back(INVALID_COLUMN_ID); projection.push_back( - LanceReader::ProjectedColumn{indicesSchema[1].name, "BIGINT", LogicalType::INT64()}); + LanceReader::ProjectedColumn{indicesSchema[1].name, LogicalType::INT64()}); projColumnIDs.push_back(INVALID_COLUMN_ID); } else { projection.push_back( - LanceReader::ProjectedColumn{indicesSchema[0].name, "BIGINT", LogicalType::INT64()}); + LanceReader::ProjectedColumn{indicesSchema[0].name, LogicalType::INT64()}); projColumnIDs.push_back(INVALID_COLUMN_ID); } for (auto i = 0u; i < relGroupEntry->getNumProperties(); ++i) { @@ -169,12 +173,12 @@ void LanceRelTable::ensureReader(Transaction* transaction, if (it == indicesSchema.end()) { continue; } - projection.push_back(LanceReader::ProjectedColumn{it->name, - LanceReader::toDuckDBCastSpelling(prop.getType()), prop.getType().copy()}); + projection.push_back(LanceReader::ProjectedColumn{it->name, prop.getType().copy()}); projColumnIDs.push_back(relGroupEntry->getColumnID(i)); } - scanState.reader = std::make_unique(connector, indicesDatasetPath, - std::move(projection), context); + scanState.dataset = lanceDataset; + scanState.reader = + std::make_unique(std::move(lanceDataset), std::move(projection)); scanState.projColumnIDs = std::move(projColumnIDs); scanState.outputToCached.assign(scanState.columnIDs.size(), -1); for (size_t outCol = 0; outCol < scanState.columnIDs.size(); ++outCol) { @@ -251,7 +255,6 @@ void LanceRelTable::initScanState(Transaction* transaction, TableScanState& scan // Range-limit the indices scan to the dataset rows that can contain edges of this // bound-node batch (CSR is sorted by source node). FWD only; BWD bound nodes are // targets scattered through the file. Mirrors IceDiskRelTable. - bool rangeLimited = false; if (layout == LanceRelTableLayout::CSR && lanceScanState.direction == RelDataDirection::FWD && !indptrData.empty() && !lanceScanState.boundNodeOffsets.empty()) { common::offset_t minNode = std::numeric_limits::max(); @@ -264,7 +267,6 @@ void LanceRelTable::initScanState(Transaction* transaction, TableScanState& scan auto endIdx = std::min((uint64_t)maxNode + 1, indptrData.size() - 1); startRow = (uint64_t)indptrData[minNode]; numRows = (uint64_t)indptrData[endIdx] - startRow; - rangeLimited = true; } } lanceScanState.currentBatchStartOffset = static_cast(startRow); @@ -291,15 +293,15 @@ void LanceRelTable::loadIndptrData(Transaction* transaction) const { } { auto context = transaction->getClientContext(); - ensureSchemasLoaded(context); - auto indptrSchema = LanceReader::fetchSchema(*connector, context, indptrDatasetPath); + auto indptrDataset = std::make_shared(indptrDatasetPath); + const auto indptrSchema = indptrDataset->schema(); if (indptrSchema.empty()) { throw RuntimeException("Indptr Lance dataset has no columns"); } std::vector projection; projection.push_back( - LanceReader::ProjectedColumn{indptrSchema[0].name, "BIGINT", LogicalType::INT64()}); - LanceReader reader(connector, indptrDatasetPath, std::move(projection), context); + LanceReader::ProjectedColumn{indptrSchema[0].name, LogicalType::INT64()}); + LanceReader reader(indptrDataset, std::move(projection)); reader.openRange(0, UINT64_MAX); auto memoryManager = MemoryManager::Get(*context); DataChunk chunk(1); @@ -563,8 +565,7 @@ row_idx_t LanceRelTable::getTotalRowCount(const Transaction* transaction) const return 0; } try { - const auto count = - static_cast(LanceReader::countRows(*connector, context, indicesDatasetPath)); + const auto count = static_cast(ensureDataset(context)->countRows()); cachedRowCount.store(count, std::memory_order_relaxed); return count; } catch (const std::exception&) { diff --git a/lance/test/lance_data/indices_follows.lance/_transactions/0-555c15fd-2050-448e-a9e0-16be92c04577.txn b/lance/test/lance_data/indices_follows.lance/_transactions/0-555c15fd-2050-448e-a9e0-16be92c04577.txn new file mode 100644 index 0000000000000000000000000000000000000000..efe7d0237bb3dfe116564593f494aaaa5c2bcf0d GIT binary patch literal 182 zcmZ9DK?=e!6huv-6}tQ@5rr;Xh!FWd<|pY9TzH6>rc#hDG{^8X9?6SnBe?MmGm9Ap zHq0D|`%Wrklv=weeH&4dImcE7)NJjudWiZ2$J!wgVGhEq%(wBXw~X5K*0IB2DWdHR z2Il?Mk0C-a6U)g>u`TdbHgi}(mHh1zQ}g~IcI7Z8qn&^?l=or!EyVwPP)>XRI!QJU literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indices_follows.lance/_transactions/0-d9417151-fab9-469f-8983-c86ca9730200.txn b/lance/test/lance_data/indices_follows.lance/_transactions/0-d9417151-fab9-469f-8983-c86ca9730200.txn deleted file mode 100644 index ffd22ea8b36cc9a41b6e4f587ff95264f74532a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 186 zcmWeqNwG9BG&eLg)J;oFveY#(vrN;qu(UANO|~#gPP8;PHZU?U*u*x0k;_-elgq*Y z2n@jx!~nAlLBJrz%q-E;#LUzp(a_AoFx4m}#UeS$*x1xiFDEfCIaP>>fl&&G6__-b v4BA;0*oEYT*h&(M(o;(m{=)#H7Hei+iLsFZqXnaoybx<~CQzpYBX$J<=2A8# diff --git a/lance/test/lance_data/indices_follows.lance/_versions/18446744073709551614.manifest b/lance/test/lance_data/indices_follows.lance/_versions/18446744073709551614.manifest index d0ac44e11deea32422d99b27839a50cb62122c46..ed657a76b9180c3dddfd106df4bb0e38a45338d8 100644 GIT binary patch literal 463 zcmdnSz`!7+VrpudY-pO6qHAPeYM^UkVv(wwXqjrDYiO2~YH5^gU}9=+zKN}ek;_-e zlgq-u(9qBTL>L-?AP9g(5{=T#O$|)U(#$PPEew*)4O0!wQ_KvKO-=Q367!N%g_sx^ zrGQw0NrTDY5vu~bkc<#pNn%lYYKg*s7+}<5&CDw?GqGS4k`-bt&IIa{U__TTHnL!3 zWMp9A!fOB*eglN4>lq0~EnzNRut(#|Qj3Z+^Yet5$_%Y|IR1WZxXvc9WB=VbOc4x8 xS_Znr1h-yEMP3#^7b_&>*bI&J4D<}Dcwhp|MtVjo7#Vf}qYna@7roY+z(yu!(H~BbTp` zCzpi*5Ez0Xhyi99f`CDanOUNxiJ7TIqM@0EVX9F|ibZmgv9YP4UQS|Oa;gv$1EUlW zD==v=8ML!1unWlvv6UnirKgrC{D%QXE!NDu5@RC+MhiwEc_G%~OrTB)M(hec0A0^T zzz{A%h6qvDQxc3;JRG;~H(X~EXjtBSnlXYQNy|W&m=M-0smROX=VFBf7Mr1wo`If0 a6%R~+*+|c51tY^gpr0XtiNVLw&lv#tn0~+j diff --git a/lance/test/lance_data/indices_follows.lance/data/0000101010000110101111110d66a94658a1681e2dd8cb3351.lance b/lance/test/lance_data/indices_follows.lance/data/0000101010000110101111110d66a94658a1681e2dd8cb3351.lance deleted file mode 100644 index 3d641689738cabdf9536390841ed84a1af375ad9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 647 zcmZ=@U|{f|05CAf08RM!j}b_MFcT0nL)jo21pWa5E-M0n^0?%P5ocf!0h+)HwdLQx z=j;p&Pk{It5I=>|Ap4n^7#SI{xP!})OHPQbB(W$xwM5}R3@~c3X6BU`8yPTKFmlNY zu@-0MC8tU-Vpkx*&ZQ}&&ZWvFub-0$RHv7kmz-@M_8W;~S0xdKETIe8T$7MsvLTN5ZDLyA2F0iFSAd&-QH515cS5O=SK|X|H*aGEO hL;2t|0A#yD`7rhRpmYHQ{!kj0RG5GX#>dgm8309SouU8$ diff --git a/lance/test/lance_data/indices_follows.lance/data/011100111101010000011100a2f75046f78580c71e07d60c55.lance b/lance/test/lance_data/indices_follows.lance/data/011100111101010000011100a2f75046f78580c71e07d60c55.lance new file mode 100644 index 0000000000000000000000000000000000000000..db32f266a4b104ba4f5f60454fa9a74910a5d60a GIT binary patch literal 738 zcmZo*U|{f|05C8Z08L;6;(uVk2xNf(6NF-5hSD%`7#}Lkz{Cgw|NfEdgaDvH43O_zwe&TCACQ zC1xfTj9jupti_pm$*B^I=+eeU7K~cLT)a7nK$jh zeXyKfYF=`FN@iYqv7U2&PHAqQE0iz9#Kj=w!^OnlAjHJtA;GTXAY{j7!zGKZ&B#zM zATd4FC$TcWv_wdnOHzu@iHD1Yi;2TQ2t;xiF)~^(d9VjCl47wR7s~<01}31@2IN?+ sfM)eHXcDP}CJlaQ5~_ys9iZ}IP(Dn37gXIUC=F5uVln}tkE5S60Gh3_N&o-= literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indices_livesin.lance/_transactions/0-901c37e7-0288-4445-8bdf-95ade946db17.txn b/lance/test/lance_data/indices_livesin.lance/_transactions/0-901c37e7-0288-4445-8bdf-95ade946db17.txn new file mode 100644 index 0000000000000000000000000000000000000000..2c2ddffb0ee86b6ae34258f1ad60cfcad0764f56 GIT binary patch literal 148 zcmXwtF%AMD5Ji`bZm}h$v7oRpCYV8n1x{e$DPS0l(QIKjnWwYUrSRn^FaOD11Vh%d z*Ki6!BcjtGm#dC06^=NSZ2Wt8j2C$xg8{P)8qMu40l;~mEFb}8MvH*XFvU!ekIys{ gOVMpar}}*lpI)uxRt`12Zd}#(k2tLFbDc1VA8B$a3;+NC literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indices_livesin.lance/_transactions/0-f43af6ed-319f-4763-be43-5356d559fb7c.txn b/lance/test/lance_data/indices_livesin.lance/_transactions/0-f43af6ed-319f-4763-be43-5356d559fb7c.txn deleted file mode 100644 index 99a6809b381320d41a08667e420615e42eb5b571..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 150 zcmXxWF%H5Y6oBDEv_nTm2gBfCOfWp&gTe_MJOvamCfdPpGVkO$w8r=^-!FERFjiNx zLI_pS=uo9Xs4QBStqnDe)3d)1XK@-l0O)UvpqKp#U{DiUa+qQ^7UGmW0RzKJcH%Uj5u z%fisW5Qq#w07QehKmh{-Q*-k~V*`_9gXE-C6Jrwt6H~*~WXq&vgCxD2#JuEGAw~u% zMg|2Y4JLyhObRSQGD2)6iACwDB?|vxfKiJzGq1$V#Dehw&|O@3RR~ejF%pbg!d$#y zXU3PM78Pga=Ls>D8Cvmh{QcT+olRiJ`%`n7A{dgi40MSJE4`A6yexh$R*1jZ42|^+ c^bD$aU;@lWdPXZ48D;?e2mwqCK8}9Q0NTN8s{jB1 literal 379 zcmbQnz`!7+l4fF@m}ZumqHAnunWk%EZf2~TlxkwEYiew2mSSpZnU-XpyooKB%Uj5u z%fism&;SI0w4niz384)Qjf@RUj7?3mj7$v^lT$2A3{5O8EKT%s67!N%g%}y6 z7#S3pG?)yYGAXbK$qBKQBo?KomMHv(0Y)v>%)Am~BLhYY#1>L`?@tFk10& z+`8XzolW4z@=0eHBN&pj40MSJ8oiQ=yexh$R){y*42|>*^bD$aU;@lWdPXZ48D;@J M1OZG8K8}9Q03q~bSpWb4 diff --git a/lance/test/lance_data/indices_livesin.lance/data/101101000001010101111000577a304c0cbe4340451ec9bc0b.lance b/lance/test/lance_data/indices_livesin.lance/data/101101000001010101111000577a304c0cbe4340451ec9bc0b.lance new file mode 100644 index 0000000000000000000000000000000000000000..5a6968e740e784b73ffe810cfbcff3ed1837654c GIT binary patch literal 376 zcmWGwU|{f|05C8}08L;9;(uTOWPkw^lxBp{OpIW8bgQ_GxnzXcN)n6GQ%e;7!vLce zYi3@FnTZ9XmM|A@PGVkiYJ6F0QE_H|o)A-+p#Te)rjR<9Dwn)ISWYiBFF8LYGcUbZ z&pAJ*G&j!`$`@kdVi5A-Vq$O*Vq!6nU{P`qvg5Mhl10~MWT+RAn4apBSeaj1A|%Zv zDaGf+!^OhI#Ni+WA~}p087-JRSOOTgL;X_;r5m8MIh2OU8$dno0Hr|`2r~hpkE5S6 E0O-Y4ssI20 literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indices_livesin.lance/data/111000001111010011110111230435463bad251acd84149894.lance b/lance/test/lance_data/indices_livesin.lance/data/111000001111010011110111230435463bad251acd84149894.lance deleted file mode 100644 index 7da59a172eeb206bd4d89a2bbb4b74819862392f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 357 zcmZ=@U|{f|05CA{08L;9;(uTOWH2xRF(Xujk%`%z1ui)uwvxo6^wbiC|1iL) z#hRH{Vr*o tjEojc9xMTjOF+H>f=UR*&;X@XpfpTg7UWwX(1p?<3WS+}(8tlw8321MRO0{u diff --git a/lance/test/lance_data/indptr_follows.lance/_transactions/0-4c1da2c1-d8ee-4b6f-a1dc-6169d17169bd.txn b/lance/test/lance_data/indptr_follows.lance/_transactions/0-4c1da2c1-d8ee-4b6f-a1dc-6169d17169bd.txn deleted file mode 100644 index 78ab0263053b2b812ac8974a0a58a8c5ec7c6513..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 147 zcmWeqF-bN|Ni<3})J?HSP1Q9?GE37H8V7`OffVEl1V9>*s{31h1|I;3=9no zfEWlsB#;T_85mfar6nd>m?Wm8CMH{&o0^&$nV48wS{Rxd>*XZoC8r88GDtBpC@^U- g8Qfq}U=@-UVlF5tQuq%8j9RRjc_qe1283Z5I{GPwb(M9Rw*n*h|DCj+3^Gxo+6V(6l4+ElX)!<;TK==;61aFm#>KP&n_#hd#mo(;T dRI^l#)oTq8ZCkT2v+H!T&p&F@p2sN?s}~QrC#e7c literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/indptr_follows.lance/_versions/18446744073709551614.manifest b/lance/test/lance_data/indptr_follows.lance/_versions/18446744073709551614.manifest index 51d695c807a863359c8870eebf394250e492d020..3fd33f209b9b84714317924191dd8def18310161 100644 GIT binary patch literal 390 zcmbQpz`!7+l4O>gVq$KXq-$)Dl&WiDoNAz(XkePGn`CHgW@Ma}YG`3%v575%%Uj5u z%fisWz`zhl7(xgT4=iG2kYZ_)WRhZPo@iu|YGh$xW|3rIWSo?2sF#zNmz*lZ$RNeY zpunWTWbm3vfmKLSh`FGoNZ~&WFlw=8=9QS4STNoNI*JRov=B9&A;G96%*6|KV0>9> zQE_H|o)A-+p%o9u->(hV*#r(8oIIB)f+0!EK$nnPO-TB$HA$v1M_23%PSy z7#JEF05K4NNFWo;Gcd3;OG`|&FiA{FO-!~lH#IdiGBL5Vv@kR^*2_uEOHLJHWRPNH zP+-zvGPuE{z$zpy#9UBPr0^dG7`0e4^Gb}33>YmKPXV39g-?MHHJu{CXvM>E>wd#^ zHh~lC&YWh9U`Wz3&?P2N^hzr7viP}JAs%EiG}1HBGpOQ$2{0S!8LePsm=5#}1TZo9 IIQlsQ0J$Jz*8l(j diff --git a/lance/test/lance_data/indptr_follows.lance/data/01100111110111001001110096faa84adeac97555244998153.lance b/lance/test/lance_data/indptr_follows.lance/data/01100111110111001001110096faa84adeac97555244998153.lance deleted file mode 100644 index 63da78fa457953fc61b745639aefeaf398640c06..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 344 zcmXqDU|{f|05C9!08RM!4@fXD0x=5^vjH(X5dQ-LLPm4Ra!Cs@7nBq!{D%QXE!NDu z5@RC+Mhiv(RxV8;buLvddHtNkyyR59)V$>Ul+3*JVm;^loYLGpS14bIiHkwViHnKB zL5PV(L4sAuOvr@Gh)Wh-n~|YjKw^5TPhw?$X^D^^7rzt>7ZZnq5hKtt57q$2g+OTg67!N%h19uJx#ac1a(bzG$@wXndFjP^&iOf| zxp}Toz7P`^gOC#!6N7^g6N`ZatCE?J36~L{lm0;OT{%22->Lun8N!c0KuCGBr;#)ipCRFwr$JNKVvEOfxdpO}0o$GPf{GHZe7^+{Bi} diff --git a/lance/test/lance_data/indptr_livesin.lance/_versions/18446744073709551614.manifest b/lance/test/lance_data/indptr_livesin.lance/_versions/18446744073709551614.manifest index d398377b151276e12a86b3cd0b6b44c01c329d6e..1fbcc642b90e9e9a52dd539e31c4684baf9080dd 100644 GIT binary patch literal 390 zcmbQpz`!7+Vr*t&W@=_+u4|lVoT_VLk!GoDVQguxo1Bon%Sp^jP8DKgkYZ#|VA5bR zc+I51DkLeyTu@S^@E-;kwOBLrO3X|w81Did#f4j1h?>rjVAK-k;srY}zAUwgN-FZQ__1}R1c z1ttw9gBwf=tU}U4%mpPy3jbk%QHwP*uf*8MfYE~S6wp~*_!J0H(gDM`F0JD*v(F#U}=|JB=0270c Hqn|SXEhAzT diff --git a/lance/test/lance_data/indptr_livesin.lance/data/000010101010100010101001eddf4d4e92b6dbff387abd017b.lance b/lance/test/lance_data/indptr_livesin.lance/data/000010101010100010101001eddf4d4e92b6dbff387abd017b.lance new file mode 100644 index 0000000000000000000000000000000000000000..06ef178bb1ef775ac65d936f3f647f3880558680 GIT binary patch literal 363 zcmWGwU|{f|05C9U08RM!4@iIkBa~)>(#%kr1xlk^#bv-HDa2e*Ql#)71{k$iGxJK! zOe`3+gt>Tg67!N%h19uJx#ac1a(bzG$@wXndFjP^&iOf| zxp}Toz7P`^gOC#!6N7^g6N`ZatCE?J36~L{lm0;OT{%22->Lun8N!c0KuUl+3*JVm;^loYLGpS14bIiHkwViHnKB zL5PV(L4sAuOvr@Gh)Wh-n~|YjKw^5TPhw?$X^D^^7rzt>7ZZnq5hKtt57q$2g+O|D&2)0VuAN2*%|CJJv!!R>2!@#;E zQ{_}uiN@1Dkt9q)BvfJ~Wj0fJI)tbHV!;s(fl3G9mxyB+WOS|2*jx&Y8mXYdNQF|^ zl=QCAuC(CI7>`DY`G(_T8G8@`udDd|1KTcY*JlEmVz7d_(-#}EY46|qYwga%fWsPs RuI=uP?(5ctd^-Ezd;kN0MauvH diff --git a/lance/test/lance_data/nodes_city.lance/_transactions/0-e868a4a2-b0e3-409f-8d58-5f152cb5b5ec.txn b/lance/test/lance_data/nodes_city.lance/_transactions/0-e868a4a2-b0e3-409f-8d58-5f152cb5b5ec.txn new file mode 100644 index 0000000000000000000000000000000000000000..93fb5a3fba1db73575546c200e30fc8734a50221 GIT binary patch literal 216 zcmYj}OA5j;7=%rHL6`rk2tpATB1Dqr!S5Mdc!o3&DQH@>Udx3ZP+LTBV1^H77+421 zXq7El#e-yu?;}I6k?AOO=_+06;GVMw$&PTyP;kygAHBaQzGRFQJ}N0=rG!Ikf{uHo zy?48_O)kKAMCf9Of^H~(&5JSQKx<#Tefq{V`kN{!`(-6WkGfOl~Ni#7vPD?T|OtwfgOEpciG%-y{N!H6r%u7xcVrF1u zk^&J5Od3oEo7okZg(QTSGE)@(!vLceYi3@Fv5^I%kgO0(USe*l1S7I6TX9KIW?s4l zlaP`SS3!P3X-;BEW`3Ro6S5+Z5oRV9jLM7*3|u%Z;KFYK7ZDZ+Q9C#!7`24Cc)@`c zUzS=_4D^~1Q<OqA)BRODsxbFo4~pUu!%&p^+h aiU%gZY@}zjf|214FbO~a6N8VVpECepB&AIN literal 522 zcmcb|z`!7+l5CljVrrR^qMK-tW~OUmX_TaEX=!Akn`B~XoN8=gY6xU(V%x&VX`WJ4D<}Dcwhp|MtVjo7#SV`V;%yS7sD=)oMo_zhw&Lq6DA28L9iy&zBup%`vWj%5~? TEQ9i3s=XnuWT>9p%PbB6LSZdN delta 208 zcmdnYzJz@N;{*?ZA1aIt4!q0^A3T{D{sF;6K@A12WG*Qorpy$D|1iL)#hRH{Vr*o< zXu-%OKY1RbG#ik~#HBKMFQXg>L|lM*@+~Hv$x2M`7$qhPG8;^eX5z|t&d2};`4EaB pj0wUhh4N2B`Grt^15|xFln>Ky!weC~fzlvlASM$K`Z)SI0|1?5C{h3b diff --git a/lance/test/lance_data/nodes_user.lance/_transactions/0-8a8b34fe-e4c9-4536-92ef-7991b093f482.txn b/lance/test/lance_data/nodes_user.lance/_transactions/0-8a8b34fe-e4c9-4536-92ef-7991b093f482.txn new file mode 100644 index 0000000000000000000000000000000000000000..4c6440f7349ebd1ec35f4bc481e3ee8327eeea84 GIT binary patch literal 209 zcmYj|O$x#=6ogG$6}o(@q6l5M5TTNv*Z6V<7oH)lX(?zH+B>;%ofzoH7zpM8}Z79>+KvX>3J^Qq|gfRJAaq zjgkTnR~H%&$&B$;jDqjDcrA+*a^U@?`25B8^F9n(5rYk+!QFh$rgZTb?tSQp!x|FT M`J7K_{2COGvZ}80=$k9hGS42D=sO@%uBam z5|S2TPE1dgU_zD!=`}O4V3c8GVBo@OG8cZ6x$v1RMC|~OVAK-k;spm!d|7Hyab|v= z5L20<6%WVXuMOAP1YW#cJew(kAxXE2+rK;^$(8gg2X^v7UjRK@|^7fZ0gT TXays~RbZq;0270cqn|SX=T@7J literal 508 zcmcc4z`!7+l4fRVoM>d3qHC0BZm4TwmTatRVPp#cyANdpiKgdhYY3{8@g4HHe0%}kS$(ku-vlT%I83{%t0jFa?o67!N%g_s!_ znWR930+R-l!CZC)79lAirpy$D|1iL)#hRH{Vr*o}!GtW2(9g@rz`%vad@e%fbKy5%h}wZ6!Dz+9aqE7=bvA(mTUQ@u zj9^I8GSDR^ZuCkj^0N53SRuj8W@w~mpl49U0~262(lc7Y$Z#7N*AT$O;N$4$3;;Tk BnOOh; diff --git a/lance/test/lance_data/nodes_user.lance/data/11010111000101010010000114cc1a4c65bbf919ce5f1ef63b.lance b/lance/test/lance_data/nodes_user.lance/data/010010000101100001101011c169584412b3ee42b068362104.lance similarity index 60% rename from lance/test/lance_data/nodes_user.lance/data/11010111000101010010000114cc1a4c65bbf919ce5f1ef63b.lance rename to lance/test/lance_data/nodes_user.lance/data/010010000101100001101011c169584412b3ee42b068362104.lance index 84ac02e0186811e13bcf4ad9ac1612507400e1f1..0d155d3157fbc330055d846630a511bbc844eab2 100644 GIT binary patch delta 246 zcmbQszJ`4PBcsDaMn`1@1_p+I|0IDV2*^Pw1`Q}}1fnN0O3QI&b4ds>Wu_?nhXF<{ z*37&TVmVxx(S+6r7TTv9?znJEhYVSrJKH8Zco*vNpJMsYTX_~y5aGK`aznXEG&Fak{if_w%)Am~BMZiRKsRyWlog_;BP1BLM7emu&WkTgEh^5; z&l6%Pv$V9d;^FxFwc$FOz=7p&<}pPuBxxA{LxXTA>6KLEW$|;dLVV0-XslRhT^^7>#|z0|zq{FKbR^kO~d{G8I43q$V7rs$v{YtONUDqUAd8=UO-}c zs!w8Nerbsiw-BcmBcl-uBLf42DG-A|B7}mIFmV_?4Jxw?N`v%*FcT2^IQlsQ0KdjC ABLDyZ literal 0 HcmV?d00001 diff --git a/lance/test/lance_data/rels_follows_flat.lance/_transactions/0-3cdb6ab2-973b-49a6-9deb-16b5d7c90f6a.txn b/lance/test/lance_data/rels_follows_flat.lance/_transactions/0-3cdb6ab2-973b-49a6-9deb-16b5d7c90f6a.txn deleted file mode 100644 index 50da8087810be7f7e12b515f260efc6b8db11be8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 216 zcmWeqF-}fNGD}P{(zP@Z=e8mAf=Tcjl=CK+2Knp&9X)RzKLxGBbUFB zHD=-9e#9#aHEl9qukG0~(~QjwR%&&3J}aW+F^Jp(<1Djt{svyq`?xQT5YBbUFB zHK(iSd=^5x5RPn$Bn2q#|RxmQ$0me22FfsTz`Z)stzayGM diff --git a/lance/test/lance_data/rels_follows_flat.lance/data/00011110001101011011010042350c446c888cc9e91705b747.lance b/lance/test/lance_data/rels_follows_flat.lance/data/00011110001101011011010042350c446c888cc9e91705b747.lance new file mode 100644 index 0000000000000000000000000000000000000000..7b83c7f45ee5d4a0f21da03bcb7881c738c9691b GIT binary patch literal 1031 zcmc&z-AV#c5T0{(ZDpv4Qx}>@5=G>Ne?{y?#FhkJi1-F2y2ygIe^UuQ=lhuHOzip5F}%)u1=L^RGEq5<%5bf}^Bqr>uQPUEn!p z7uzc3LbKI%>#cDx9H!r(L|Yalzt~GLfEoqHN`T*4KycWDoDEa57JXPF-W?! tiga$_t4sX^1wK==rUt>kqp9p3KsEX8C-HOcKe0RPUsX zSQb^w!$rUsWDaqJ8htE}$o>tzN-LQ$RsnHQV2y!dBp6uwk8gcetH&Bl99*mQXWdO4 eJn8&V=R@7^>wJ3uO0VpL_Nq;Vf*<#+R`~>F^6N Date: Mon, 26 Oct 2026 22:49:25 -0700 Subject: [PATCH 3/6] docs(lance): note vector-capacity requirement --- lance/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lance/README.md b/lance/README.md index 27c29265..b1544bdd 100644 --- a/lance/README.md +++ b/lance/README.md @@ -120,5 +120,7 @@ version-mismatch error test). configured later) and fail at first scan, mirroring icebug-disk. - Semi-mask morsel skipping is not implemented (masks still filter); GDS vertex scans (`scanVertices`) over Lance tables are not supported. +- Scan batches are sized for the default engine vector capacity (2048 rows); + custom `LBUG_VECTOR_CAPACITY_LOG2` builds below 11 are not supported. - `LOAD` the extension before creating Lance tables; the "load the extension" error otherwise points at the fix. From efa21537ba73b229a55e1cc5b911dc1bdda4c99a Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Tue, 27 Oct 2026 07:20:32 -0700 Subject: [PATCH 4/6] fix(ci): install Rust toolchain and protoc for lance native reader The manylinux container has no cargo, so the lance_ladybug_ffi custom command died mid-build with a bare 'no such file or directory'. Install rustup stable plus the matrix-arch std target, and ensure protoc exists (prost-build needs it) via dnf with an official-release fallback. CMake now locates cargo at configure time and fails fast with a clear message instead of a mid-build custom-command error. --- .github/workflows/ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ lance/CMakeLists.txt | 15 +++++++++++---- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a69a3a5f..2878deec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,6 +228,44 @@ jobs: test -e /usr/lib64/libarrow.so.2600 ldconfig -p | grep -q 'libarrow.so.2600' + # The lance extension builds a native reader (lance_ladybug_ffi) via + # Cargo. The manylinux container ships no Rust toolchain and no + # protoc (needed by prost-build through lance-core), so install both + # here before the CMake configure step. + - name: Install Rust toolchain and protoc (Linux) + if: runner.os == 'Linux' + run: | + set -e + if ! command -v cargo >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --profile minimal --default-toolchain stable + fi + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + export PATH="$HOME/.cargo/bin:$PATH" + cargo --version + # CMake maps matrix.arch to the Rust target triple; make sure the + # matching std target is installed (default toolchain already + # carries the native triple). + case "${{ matrix.arch }}" in + x86_64) rustup target add x86_64-unknown-linux-gnu ;; + aarch64|arm64) rustup target add aarch64-unknown-linux-gnu ;; + esac + if ! command -v protoc >/dev/null 2>&1; then + if ! dnf install -y protobuf-compiler; then + # Fallback: official protoc release (covers repos without the + # compiler package). + PROTOC_VERSION="29.3" + case "${{ matrix.arch }}" in + x86_64) PROTOC_ARCH="x86_64" ;; + aarch64|arm64) PROTOC_ARCH="aarch_64" ;; + esac + curl -LsSf -o /tmp/protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip" + unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' + rm -f /tmp/protoc.zip + fi + fi + protoc --version + - name: Shim libomp.so.5 -> installed libomp (Linux) if: runner.os == 'Linux' run: | diff --git a/lance/CMakeLists.txt b/lance/CMakeLists.txt index ad3b2500..f8f44714 100644 --- a/lance/CMakeLists.txt +++ b/lance/CMakeLists.txt @@ -87,6 +87,13 @@ file(GLOB_RECURSE RUST_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/rust set(RUST_FFI_DEPENDS ${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml ${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.lock ${RUST_SOURCES}) +# Fail fast at configure time when cargo is unavailable instead of dying +# mid-build with a bare "no such file or directory" from the custom command. +find_program(CARGO_EXECUTABLE cargo) +if(NOT CARGO_EXECUTABLE) + message(FATAL_ERROR "cargo is required to build the lance extension (native Lance reader). Install the Rust toolchain, e.g. via rustup: https://rustup.rs") +endif() + set(RUST_CARGO_ENV CARGO_TARGET_DIR=${RUST_TARGET_DIR} MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}) if(NOT "$ENV{PROTOC}" STREQUAL "") @@ -123,8 +130,8 @@ endif() if(CMAKE_CONFIGURATION_TYPES) add_custom_command( OUTPUT "${RUST_DEBUG_LIB}" "${RUST_RELEASE_LIB}" - COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} cargo build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --target=${RUST_PLATFORM_TARGET} - COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} cargo build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --release --target=${RUST_PLATFORM_TARGET} + COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} ${CARGO_EXECUTABLE} build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --target=${RUST_PLATFORM_TARGET} + COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} ${CARGO_EXECUTABLE} build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --release --target=${RUST_PLATFORM_TARGET} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} DEPENDS ${RUST_FFI_DEPENDS} VERBATIM) @@ -136,7 +143,7 @@ else() if(CMAKE_BUILD_TYPE STREQUAL "Debug") add_custom_command( OUTPUT "${RUST_DEBUG_LIB}" - COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} cargo build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --target=${RUST_PLATFORM_TARGET} + COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} ${CARGO_EXECUTABLE} build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --target=${RUST_PLATFORM_TARGET} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} DEPENDS ${RUST_FFI_DEPENDS} VERBATIM) @@ -144,7 +151,7 @@ else() else() add_custom_command( OUTPUT "${RUST_RELEASE_LIB}" - COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} cargo build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --release --target=${RUST_PLATFORM_TARGET} + COMMAND ${CMAKE_COMMAND} -E env ${RUST_CARGO_ENV} ${CARGO_EXECUTABLE} build --manifest-path=${CMAKE_CURRENT_LIST_DIR}/rust/Cargo.toml --release --target=${RUST_PLATFORM_TARGET} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} DEPENDS ${RUST_FFI_DEPENDS} VERBATIM) From e3ac843d1b6a73b52b1444e75d96a063b5b585af Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Tue, 27 Oct 2026 08:18:20 -0700 Subject: [PATCH 5/6] fix(lance): drop project() so the extension artifact lands in build dir Calling project() redefined PROJECT_SOURCE_DIR to extension/lance, so build_extension_lib() wrote liblance.lbug_extension to extension/lance/extension/lance/build instead of extension/lance/build where LOAD_DYNAMIC_EXTENSION resolves it. All 9 lance e2e cases failed with 'neither an official extension, nor does the extension path exist'. Other extensions never call project(); match that convention. --- lance/CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lance/CMakeLists.txt b/lance/CMakeLists.txt index f8f44714..915f75c4 100644 --- a/lance/CMakeLists.txt +++ b/lance/CMakeLists.txt @@ -1,11 +1,15 @@ -cmake_minimum_required(VERSION 3.14...3.29) - # Set extension name here set(TARGET_NAME lance) set(EXTENSION_NAME ${TARGET_NAME}_extension) -project(${TARGET_NAME}) +# NOTE: no project() call here (unlike a standalone build): defining a new +# project would redefine PROJECT_SOURCE_DIR to this directory, and the +# shared build_extension_lib() helper locates the extension output at +# ${PROJECT_SOURCE_DIR}/extension//build. With project() the artifact +# would land at extension/lance/extension/lance/build instead of +# extension/lance/build, where the test runner's LOAD_DYNAMIC_EXTENSION +# expects it. include_directories(src/include) # Build and link the Rust staticlib (lance_ladybug_ffi), following the From 1225a35a27deb3b3737a0b6075168f64e14bfa0f Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Sun, 1 Nov 2026 11:05:33 -0800 Subject: [PATCH 6/6] feat(lance): migrate to generalized EXTERNAL storage hooks (ladybug#1456) Core generalized per-format storage beyond LANCE (ladybugdb/ladybug#1456): StorageFormat::LANCE is gone, replaced by EXTERNAL plus a persisted format name; the hook surface is renamed to ColumnarTableHooks / ColumnarTableHookRegistry, registered by name, with supportsMixedColumnarScan replacing the scanStateRank lattice and ColumnarRelTableScanState as the shared multi-rel scan-state base. - Register under "lance" with supportsMixedColumnarScan instead of StorageFormat::LANCE + scanStateRank 3. - LanceRelTableScanState extends ColumnarRelTableScanState directly (was IceDiskRelTableScanState, now an empty subclass kept for compat). --- lance/README.md | 8 ++++---- lance/src/include/storage/lance_node_table.h | 2 +- lance/src/include/storage/lance_rel_table.h | 13 ++++++------- lance/src/main/lance_extension.cpp | 15 ++++++++------- lance/test/test_files/lance.test | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lance/README.md b/lance/README.md index b1544bdd..3f37bb8f 100644 --- a/lance/README.md +++ b/lance/README.md @@ -72,14 +72,14 @@ function (graph traversal needs CSR extend, degree summaries, PK lookup), so core provides a small extension hook surface (see `src/include/storage/table/external_table_hooks.h`, LadybugDB/ladybug#1445): -- `StorageFormat::LANCE` + `ExternalTableHookRegistry` (factories + scan hooks, - keyed by format and by table); +- `ColumnarTableHookRegistry` (factories + scan hooks, registered under the + `"lance"` external format name and keyed by table); - the scan operators (`ScanNodeTable`, `ScanRelTable`, `ScanMultiRelTable`, `OnDiskGraph` for recursive traversal) create scan states and count morsels through the hooks, falling back to the legacy paths otherwise; -- `LanceRelTableScanState` extends `IceDiskRelTableScanState` so one shared +- `LanceRelTableScanState` extends `ColumnarRelTableScanState` so one shared state can scan lance, icebug-disk, Arrow, and native tables in multi-rel - scans (rank 3 in the coverage lattice). + scans (`supportsMixedColumnarScan`). No core query path changed behavior: all pre-existing icebug-disk/Arrow/native tests pass unmodified. diff --git a/lance/src/include/storage/lance_node_table.h b/lance/src/include/storage/lance_node_table.h index a7d956f7..d1e17535 100644 --- a/lance/src/include/storage/lance_node_table.h +++ b/lance/src/include/storage/lance_node_table.h @@ -83,7 +83,7 @@ class LanceNodeTable final : public storage::ColumnarNodeTableBase { uint64_t vectorPos, common::offset_t& result) const override; const std::string& getDatasetPath() const { return datasetPath; } - // Morsel count for ScanNodeTableSharedState::initialize (via ExternalTableHooks). + // Morsel count for ScanNodeTableSharedState::initialize (via ColumnarTableHooks). common::node_group_idx_t getNumScanMorsels(const transaction::Transaction* transaction) const; protected: diff --git a/lance/src/include/storage/lance_rel_table.h b/lance/src/include/storage/lance_rel_table.h index 6b0a5ac5..e817920a 100644 --- a/lance/src/include/storage/lance_rel_table.h +++ b/lance/src/include/storage/lance_rel_table.h @@ -9,7 +9,6 @@ #include "common/types/internal_id_util.h" #include "storage/lance_reader.h" #include "storage/table/columnar_rel_table_base.h" -#include "storage/table/ice_disk_rel_table.h" #include "transaction/transaction.h" namespace lbug { @@ -17,11 +16,11 @@ namespace lance_extension { enum class LanceRelTableLayout : uint8_t { CSR, FLAT }; -// Extends IceDiskRelTableScanState (rather than RelTableScanState directly) so a single +// Extends ColumnarRelTableScanState (rather than RelTableScanState directly) so a single // shared scan state can serve lance, icebug-disk, Arrow, and native rel tables in -// multi-rel scans; see ExternalTableHooks::scanStateRank. The Parquet reader members -// inherited from the icebug-disk state stay null; Lance reads go through `reader`. -struct LanceRelTableScanState final : storage::IceDiskRelTableScanState { +// multi-rel scans; see ColumnarTableHooks::supportsMixedColumnarScan. The Parquet reader +// members inherited from the columnar base stay null; Lance reads go through `reader`. +struct LanceRelTableScanState final : storage::ColumnarRelTableScanState { std::shared_ptr dataset; std::unique_ptr reader; // Maps scan-output positions to cached-batch positions (catalog column ID order is @@ -34,9 +33,9 @@ struct LanceRelTableScanState final : storage::IceDiskRelTableScanState { LanceRelTableScanState(storage::MemoryManager& mm, common::ValueVector* nodeIDVector, std::vector outputVectors, std::shared_ptr outChunkState) - : storage::IceDiskRelTableScanState{mm, nodeIDVector, std::move(outputVectors), + : storage::ColumnarRelTableScanState{mm, nodeIDVector, std::move(outputVectors), std::move(outChunkState)} {} - // setToTable is inherited from IceDiskRelTableScanState: CSR column resolution against + // setToTable is inherited from ColumnarRelTableScanState: CSR column resolution against // the (empty) RelTableData scaffolding is safe for Lance tables exactly as for // icebug-disk tables; the Lance scan reads through `reader` instead. }; diff --git a/lance/src/main/lance_extension.cpp b/lance/src/main/lance_extension.cpp index 0c91ce9a..8e418518 100644 --- a/lance/src/main/lance_extension.cpp +++ b/lance/src/main/lance_extension.cpp @@ -13,10 +13,12 @@ namespace lance_extension { void LanceExtension::load(main::ClientContext* context) { auto& db = *context->getDatabase(); extension::ExtensionUtils::addTableFunc(db); - // Register Lance-backed table storage for StorageFormat::LANCE. The scan-state - // rank (3) declares that Lance scan states extend the icebug-disk scan state and - // therefore cover every lower-rank backend in shared multi-rel scans. - storage::ExternalTableHooks hooks; + // Register Lance-backed table storage under the "lance" external format name. + // supportsMixedColumnarScan declares that Lance scan states extend + // ColumnarRelTableScanState and therefore cover the in-core columnar backends + // in shared multi-rel scans. + storage::ColumnarTableHooks hooks; + hooks.formatName = "lance"; hooks.createNodeTable = [](const storage::StorageManager* storageManager, const catalog::NodeTableCatalogEntry* nodeTableEntry, storage::MemoryManager* memoryManager, main::ClientContext* ctx) { @@ -47,9 +49,8 @@ void LanceExtension::load(main::ClientContext* context) { return std::make_unique(*memoryManager, boundNodeIDVector, outVectors, std::move(outChunkState)); }; - hooks.scanStateRank = 3; - storage::ExternalTableHookRegistry::registerHooks(common::StorageFormat::LANCE, - std::move(hooks)); + hooks.supportsMixedColumnarScan = true; + storage::ColumnarTableHookRegistry::registerHooks("lance", std::move(hooks)); } } // namespace lance_extension diff --git a/lance/test/test_files/lance.test b/lance/test/test_files/lance.test index 923ba5f9..30dd1d89 100644 --- a/lance/test/test_files/lance.test +++ b/lance/test/test_files/lance.test @@ -201,7 +201,7 @@ Runtime exception: Lance dataset '${LBUG_ROOT_DIRECTORY}/extension/lance/test/la -LOG UnknownFormatFails -STATEMENT CREATE NODE TABLE t4(id INT64 PRIMARY KEY) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data', format = 'foo') ---- error -Binder exception: Unsupported storage format 'foo'. Valid options are: icebug-disk, lance. +Runtime exception: Unsupported storage format option for node table: foo. If this format is provided by a Ladybug extension, load the extension and try again. -LOG WrongVersionFails -STATEMENT CREATE NODE TABLE t5(id INT32 PRIMARY KEY) WITH (storage = '${LBUG_ROOT_DIRECTORY}/extension/lance/test/lance_data/nodes_wrongversion.lance', format = 'lance')