diff --git a/common/vault/data/__init__.py b/common/vault/data/__init__.py index 23b585aa..61c2e426 100644 --- a/common/vault/data/__init__.py +++ b/common/vault/data/__init__.py @@ -1,2 +1,10 @@ from ._base_insert_request import BaseInsertRequest from ._base_insert_response import BaseInsertResponse +from ._base_query_request import BaseQueryRequest +from ._base_query_response import BaseQueryResponse +from ._base_get_tokens_request import BaseGetTokensRequest +from ._base_get_tokens_response import BaseGetTokensResponse +from ._base_upload_files_request import BaseUploadFilesRequest +from ._base_upload_files_response import BaseUploadFilesResponse +from ._base_delete_files_request import BaseDeleteFilesRequest +from ._base_delete_files_response import BaseDeleteFilesResponse diff --git a/common/vault/data/_base_delete_files_request.py b/common/vault/data/_base_delete_files_request.py new file mode 100644 index 00000000..063cc977 --- /dev/null +++ b/common/vault/data/_base_delete_files_request.py @@ -0,0 +1,3 @@ +class BaseDeleteFilesRequest: + def __init__(self, records): + self.records = records diff --git a/common/vault/data/_base_delete_files_response.py b/common/vault/data/_base_delete_files_response.py new file mode 100644 index 00000000..c4945bcd --- /dev/null +++ b/common/vault/data/_base_delete_files_response.py @@ -0,0 +1,9 @@ +class BaseDeleteFilesResponse: + def __init__(self, records=None): + self.records = records + + def __repr__(self): + return f"{type(self).__name__}(records={self.records})" + + def __str__(self): + return self.__repr__() diff --git a/common/vault/data/_base_get_tokens_request.py b/common/vault/data/_base_get_tokens_request.py new file mode 100644 index 00000000..5e971535 --- /dev/null +++ b/common/vault/data/_base_get_tokens_request.py @@ -0,0 +1,3 @@ +class BaseGetTokensRequest: + def __init__(self, records): + self.records = records diff --git a/common/vault/data/_base_get_tokens_response.py b/common/vault/data/_base_get_tokens_response.py new file mode 100644 index 00000000..85a6fa7d --- /dev/null +++ b/common/vault/data/_base_get_tokens_response.py @@ -0,0 +1,9 @@ +class BaseGetTokensResponse: + def __init__(self, records=None): + self.records = records + + def __repr__(self): + return f"{type(self).__name__}(records={self.records})" + + def __str__(self): + return self.__repr__() diff --git a/common/vault/data/_base_query_request.py b/common/vault/data/_base_query_request.py new file mode 100644 index 00000000..842ef4fd --- /dev/null +++ b/common/vault/data/_base_query_request.py @@ -0,0 +1,3 @@ +class BaseQueryRequest: + def __init__(self, query): + self.query = query diff --git a/common/vault/data/_base_query_response.py b/common/vault/data/_base_query_response.py new file mode 100644 index 00000000..11ee2fe5 --- /dev/null +++ b/common/vault/data/_base_query_response.py @@ -0,0 +1,9 @@ +class BaseQueryResponse: + def __init__(self, records=None): + self.records = records + + def __repr__(self): + return f"{type(self).__name__}(records={self.records})" + + def __str__(self): + return self.__repr__() diff --git a/common/vault/data/_base_upload_files_request.py b/common/vault/data/_base_upload_files_request.py new file mode 100644 index 00000000..b0ac57a5 --- /dev/null +++ b/common/vault/data/_base_upload_files_request.py @@ -0,0 +1,3 @@ +class BaseUploadFilesRequest: + def __init__(self, records): + self.records = records diff --git a/common/vault/data/_base_upload_files_response.py b/common/vault/data/_base_upload_files_response.py new file mode 100644 index 00000000..d156afb9 --- /dev/null +++ b/common/vault/data/_base_upload_files_response.py @@ -0,0 +1,9 @@ +class BaseUploadFilesResponse: + def __init__(self, records=None): + self.records = records + + def __repr__(self): + return f"{type(self).__name__}(records={self.records})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/README.md b/flowvault/README.md index e2f92883..81d16d88 100644 --- a/flowvault/README.md +++ b/flowvault/README.md @@ -1,6 +1,6 @@ # Skyflow FlowVault Python SDK -The `flowvault` module is a Skyflow Python SDK for vault operations. It shares its client, credentials, and configuration with the [skyvault SDK](../skyvault/README.md) (both depend on the `common` module) but exposes a different, narrower surface: **unary** vault operations — insert, get, update, delete, and detokenize. +The `flowvault` module is a Skyflow Python SDK for vault operations. It shares its client, credentials, and configuration with the [skyvault SDK](../skyvault/README.md) (both depend on the `common` module) but exposes a different, narrower surface: **unary** vault operations — insert, get, update, delete, detokenize, query, get tokens, upload files, and delete files. > Meant for **FlowVault** vaults. @@ -38,6 +38,10 @@ The `flowvault` module is a Skyflow Python SDK for vault operations. It shares i - [Get](#get) - [Update](#update) - [Delete](#delete) +- [Query](#query) +- [Get Tokens](#get-tokens) +- [Upload Files](#upload-files) +- [Delete Files](#delete-files) - [Custom Request Headers](#custom-request-headers) - [Error Handling](#error-handling) - [Two layers of errors](#two-layers-of-errors) @@ -50,7 +54,7 @@ The `flowvault` module is a Skyflow Python SDK for vault operations. It shares i # Overview - Authenticate using a Skyflow service account, an API key, or a bearer token — see [Authenticate](#authenticate). -- Perform unary Vault API operations — insert, get, update, delete, and detokenize — a single API call each. See [VaultController — Unary operations](#vaultcontroller--unary-operations). +- Perform Vault API operations — insert, get, update, delete, detokenize, query, get tokens, upload files, and delete files. See [VaultController — Unary operations](#vaultcontroller--unary-operations). - **Per-record reporting, not all-or-nothing.** A call succeeds as a call even when individual records fail; every response reports the outcome of each individual record or token. See [Error Handling](#error-handling). # Install @@ -358,11 +362,11 @@ The SDK logs at `LogLevel.ERROR` by default. Levels rank `DEBUG` < `INFO` < `WAR ## Schema vs. schemaless vaults -Which operations make sense depends on whether the vault is **structured** (has a schema — tables and columns) or **schemaless** (stores standalone tokens with no table structure): the record operations (`insert`, `get`, `update`, `delete`) address a table's columns and so need a structured vault, while `detokenize` needs only the token itself and works against either kind. See [Vault type support](#vault-type-support) for the per-operation breakdown. +Which operations make sense depends on whether the vault is **structured** (has a schema — tables and columns) or **schemaless** (stores standalone tokens with no table structure): the record and file operations (`insert`, `get`, `update`, `delete`, `query`, `upload_files`, `delete_files`) address a table's columns and so need a structured vault, while the token operations (`detokenize`, `get_tokens`) need only the token and work against either kind. See [Vault type support](#vault-type-support) for the per-operation breakdown. # VaultController — Unary operations -`VaultController` is returned by `skyflow_client.vault(...)` and exposes these vault operations. Each sends exactly one API call and hands the result straight back: +`VaultController` is returned by `skyflow_client.vault(...)` and exposes these vault operations. Each is a single method call that hands the result straight back (`upload_files` orchestrates the signed-URL upload across a couple of calls internally): | Method | Parameters | Returns | Description | |--------|-----------|---------|-------------| @@ -371,12 +375,16 @@ Which operations make sense depends on whether the vault is **structured** (has | `get(request)` | `GetRequest`, optional `GetOptions` | `GetResponse` | Read records by skyflow ID or unique value, optionally with a redaction override per column | | `update(request)` | `UpdateRequest`, optional `UpdateOptions` | `UpdateResponse` | Update records by skyflow ID | | `delete(request)` | `DeleteRequest`, optional `DeleteOptions` | `DeleteResponse` | Delete records by skyflow ID or unique value | +| `query(request)` | `QueryRequest`, optional `QueryOptions` | `QueryResponse` | Run a read-only SQL `SELECT`; returns up to 25 rows | +| `get_tokens(request)` | `GetTokensRequest`, optional `GetTokensOptions` | `GetTokensResponse` | Look up existing tokens for values (deterministic token groups) | +| `upload_files(request)` | `UploadFilesRequest`, optional `UploadFilesOptions` | `UploadFilesResponse` | Upload files to file columns (the SDK fetches signed URLs and uploads the bytes) | +| `delete_files(request)` | `DeleteFilesRequest`, optional `DeleteFilesOptions` | `DeleteFilesResponse` | Delete files from file columns by skyflow ID or unique value | -Each method also accepts an optional options object (`InsertOptions`, `DetokenizeOptions`, `GetOptions`, `UpdateOptions`, `DeleteOptions`) — see [Custom Request Headers](#custom-request-headers). +Each method also accepts an optional options object (`InsertOptions`, `DetokenizeOptions`, `GetOptions`, `UpdateOptions`, `DeleteOptions`, `QueryOptions`, `GetTokensOptions`, `UploadFilesOptions`, `DeleteFilesOptions`) — see [Custom Request Headers](#custom-request-headers). ## Vault type support -The same distinction as [Schema vs. schemaless vaults](#schema-vs-schemaless-vaults) applies. Four of the five unary operations address records inside a table, so they only make sense against a structured vault: +The same distinction as [Schema vs. schemaless vaults](#schema-vs-schemaless-vaults) applies. Most operations address records inside a table, so they only make sense against a structured vault; the token-centric ones work against either kind: | Operation | Supported on | |---|---| @@ -385,6 +393,10 @@ The same distinction as [Schema vs. schemaless vaults](#schema-vs-schemaless-vau | `update` | Structured vaults — updates a table's records by skyflow ID. | | `delete` | Structured vaults — deletes a table's records. | | `detokenize` | Both — detokenizing only needs the token itself, not a table, so it works regardless of which kind of vault the token came from. | +| `query` | Structured vaults — runs a SQL `SELECT` over a table. | +| `get_tokens` | Both — looks up a token for a value within a token group, not a table. | +| `upload_files` | Structured vaults — uploads into a table's file columns. | +| `delete_files` | Structured vaults — deletes files from a table's file columns. | # Insert @@ -718,6 +730,213 @@ for record in response.records: print(record.skyflow_id, 'failed', record.http_code, record.error) ``` +# Query + +Run a read-only SQL `SELECT` against the vault in a single API call. + +> **Vault type supported:** structured (schema) vaults. See [Vault type support](#vault-type-support). + +**Note:** + +- `query` is required and must be a non-empty string. Only `SELECT` is supported. +- A call returns at most 25 records; page through more with the SQL `OFFSET` keyword. +- Returned values may be masked per your vault's policies; they are never tokens or file URLs. +- Unlike the other operations, `query` has **no per-record results** — any failure is thrown as a `SkyflowError`, and `request_id` is always populated. + +```python +from skyflow.vault.data import QueryRequest + +vault = skyflow_client.vault('') # skyflow_client from Quickstart + +request = QueryRequest(query="SELECT name, email FROM table1 WHERE name = 'john doe' LIMIT 25") +response = vault.query(request) +``` + +There is no async variant: `query` returns its `QueryResponse` directly. + +Sample response: + +```json +{ + "records": [ + { "data": { "skyflow_id": "9fac9201-7b8a-4446-93f8-5244e1213bd1", "name": "john doe", "email": "john@example.com" } } + ], + "metadata": { "columns": ["skyflow_id", "name", "email"] }, + "request_id": "a1b2c3d4-..." +} +``` + +`QueryResponse` carries `.records` (a list of `QueryResponseRecord`, each with `.data` — a dict of column to value), `.metadata` (`QueryResponseMetadata` with `.columns`), and `.request_id` (always populated). + +```python +for record in response.records: + print(record.data) +``` + +# Get Tokens + +Look up the existing token for one or more values — one result per value — in a single API call. + +> **Vault type supported:** both. See [Vault type support](#vault-type-support). + +**Note:** + +- `records` is required and must be a non-empty list of `GetTokensRequestRecord`. +- Each record needs a non-null `value` and a non-blank `token_group_name`. Only **deterministic** token groups are supported. +- One result is returned per input value, in request order; duplicate inputs are not collapsed. + +```python +from skyflow.vault.data import GetTokensRequest, GetTokensRequestRecord + +vault = skyflow_client.vault('') # skyflow_client from Quickstart + +request = GetTokensRequest(records=[ + GetTokensRequestRecord(value='john@example.com', token_group_name='email_cg'), + GetTokensRequestRecord(value='jane@example.com', token_group_name='email_cg'), +]) +response = vault.get_tokens(request) +``` + +There is no async variant: `get_tokens` returns its `GetTokensResponse` directly. + +Sample response: + +```json +{ + "records": [ + { "value": "john@example.com", "token_group_name": "email_cg", "token": "1R9kNnLOPM", "http_code": 200, "error": null, "request_id": null }, + { "value": "unknown@example.com", "token_group_name": "email_cg", "token": null, "http_code": 404, "error": "Token not found.", "request_id": "a1b2c3d4-..." } + ] +} +``` + +Accessors on each `GetTokensResponseRecord`: `.value`, `.token_group_name`, `.token`, `.http_code`, `.error`, `.request_id`. + +```python +for record in response.records: + if record.error is None: + print(record.value, '->', record.token) + else: + print(record.value, 'failed', record.http_code, record.error) +``` + +# Upload Files + +Upload one or more files to file columns. The SDK **orchestrates** this in two steps: it requests a signed upload URL per column, then uploads each file's bytes to its URL — you just supply the files. + +> **Vault type supported:** structured (schema) vaults. See [Vault type support](#vault-type-support). + +**Note:** + +- `records` is required and must be a non-empty list of `UploadFilesRequestRecord`; each needs a `table_name` and a non-empty `columns` list. +- Each `UploadFilesRequestColumn` needs a `column` name and **exactly one** file source — `file_path`, `base64`, or `file_object`. `base64` additionally requires `file_name`. `content_type` is inferred from the file name when omitted. +- Omit `skyflow_id` to create a new record (CREATE permission); set it to upload into an existing record (UPDATE permission). +- `file_name` is derived from `file_path`/`file_object` when omitted; if still unset, the server generates one. +- Uploads are **sequential**, and the final virus-scan status (`fileStatus`) is asynchronous — read it later with `get()`. + +```python +from skyflow.vault.data import UploadFilesRequest, UploadFilesRequestRecord, UploadFilesRequestColumn + +vault = skyflow_client.vault('') # skyflow_client from Quickstart + +request = UploadFilesRequest(records=[ + UploadFilesRequestRecord(table_name='onboarding', columns=[ + UploadFilesRequestColumn(column='resumePDF', file_path='/path/to/resume.pdf'), + UploadFilesRequestColumn(column='photoID', base64='', file_name='photo.jpg'), + ]), +]) +response = vault.upload_files(request) +``` + +Sample response (partial success): + +```json +{ + "records": [ + { + "skyflow_id": "2a62a1fd-...", + "table_name": "onboarding", + "columns": [ + { "column": "resumePDF", "file_name": "resume.pdf", "upload_status": "UPLOADED", "error": null }, + { "column": "photoID", "file_name": "photo.jpg", "upload_status": "FAILED", "error": "PUT failed: 403" } + ], + "http_code": 200, + "error": null, + "request_id": null + } + ] +} +``` + +Accessors on each `UploadFilesResponseRecord`: `.skyflow_id`, `.table_name`, `.columns`, `.http_code`, `.error`, `.request_id`. Each entry in `.columns` is an `UploadFilesColumnResult` with `.column`, `.file_name`, `.upload_status` (`"UPLOADED"`, `"FAILED"`, or `"SKIPPED"`), and `.error`. + +```python +for record in response.records: + for column in (record.columns or []): + print(column.column, column.upload_status, column.error) +``` + +# Delete Files + +Delete files from file columns of existing records — and remove the stored objects — in a single API call. + +> **Vault type supported:** structured (schema) vaults. See [Vault type support](#vault-type-support). + +**Note:** + +- `records` is required and must be a non-empty list of `DeleteFilesRequestRecord`; each needs a `table_name` and a non-empty `columns` list of file-column names. +- Each record sets **exactly one** of `skyflow_id` or `unique_values` — both, or neither, fails validation. +- `unique_values` takes the same shape as in [Get](#get): a list of dicts, each holding the unique column-name/value pairs that identify a record; each may resolve to more than one record. + +```python +from skyflow.vault.data import DeleteFilesRequest, DeleteFilesRequestRecord + +vault = skyflow_client.vault('') # skyflow_client from Quickstart + +request = DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name='onboarding', columns=['resumePDF'], skyflow_id='9fac9201-...'), + DeleteFilesRequestRecord(table_name='onboarding', columns=['photoID'], unique_values=[{'email': 'jane.doe@example.com'}]), +]) +response = vault.delete_files(request) +``` + +There is no async variant: `delete_files` returns its `DeleteFilesResponse` directly. + +Sample response (partial success): + +```json +{ + "records": [ + { + "skyflow_id": "9fac9201-...", + "table_name": "onboarding", + "columns": [ { "column": "resumePDF", "status": "DELETED" } ], + "http_code": 200, + "error": null, + "request_id": null + }, + { + "skyflow_id": "invalid-id", + "table_name": "onboarding", + "columns": null, + "http_code": 404, + "error": "Invalid request. skyflowID invalid-id is invalid.", + "request_id": "a1b2c3d4-..." + } + ] +} +``` + +Accessors on each `DeleteFilesResponseRecord`: `.skyflow_id`, `.table_name`, `.columns`, `.http_code`, `.error`, `.request_id`. On a successful record `.columns` is a list of `DeleteFilesColumnResult` (`.column`, `.status` — `"DELETED"`); on a failed record `.columns` is `None`. + +```python +for record in response.records: + if record.error is None: + print(record.skyflow_id, 'deleted', [c.column for c in record.columns]) + else: + print(record.skyflow_id, 'failed', record.http_code, record.error) +``` + # Custom Request Headers To include custom HTTP headers on an outgoing request, pass an **interceptor** via that operation's options object. The interceptor is a callable that receives a `RequestContext` and can add headers to it. The headers available are defined by the `CustomHeaderKey` enum: @@ -748,6 +967,10 @@ The same pattern applies to every operation, via its corresponding options class | `get` | `GetOptions` | | `update` | `UpdateOptions` | | `delete` | `DeleteOptions` | +| `query` | `QueryOptions` | +| `get_tokens` | `GetTokensOptions` | +| `upload_files` | `UploadFilesOptions` | +| `delete_files` | `DeleteFilesOptions` | # Error Handling @@ -774,7 +997,7 @@ Every response exposes `.records`. The records list has one entry per submitted | `.error` | always | Error message for this item, populated only on failure — `None` means this item succeeded. | | `.request_id` | always | The `x-request-id` of the call this item was part of, populated only on failure (`None` on success) — quote it in support escalations. | -The success payload sits alongside those attributes on the same object: `.skyflow_id`/`.tokens`/`.data`/`.hashed_data` for record-shaped operations (`insert`, `get`, `update`), `.value`/`.token_group_name`/`.metadata` for detokenize, `.skyflow_id` alone for delete. +The success payload sits alongside those attributes on the same object: `.skyflow_id`/`.tokens`/`.data`/`.hashed_data` for record-shaped operations (`insert`, `get`, `update`), `.value`/`.token_group_name`/`.metadata` for detokenize, `.value`/`.token_group_name`/`.token` for `get_tokens`, `.skyflow_id`/`.table_name`/`.columns` for `upload_files` and `delete_files`, and `.skyflow_id` alone for delete. (`query` is the exception — it has no per-record results; any failure is raised as a `SkyflowError`.) A response has just `.records`, in submitted order, with `.http_code`, `.error`, and `.request_id` on each entry alongside that operation's payload. diff --git a/flowvault/api-report/skyflow.api.json b/flowvault/api-report/skyflow.api.json index 984c4b34..96fe8e23 100644 --- a/flowvault/api-report/skyflow.api.json +++ b/flowvault/api-report/skyflow.api.json @@ -34,16 +34,24 @@ "skyflow.utils.enums.EnvUrls.PROD": "attr = 'skyvault.skyflowapis.com'", "skyflow.utils.enums.EnvUrls.SANDBOX": "attr = 'skyvault.skyflowapis-preview.com'", "skyflow.utils.enums.EnvUrls.STAGE": "attr = 'skyvault.skyflowapis.tech'", + "skyflow.utils.enums.FileUploadStatus": "class (Enum)", + "skyflow.utils.enums.FileUploadStatus.FAILED": "attr = 'FAILED'", + "skyflow.utils.enums.FileUploadStatus.SKIPPED": "attr = 'SKIPPED'", + "skyflow.utils.enums.FileUploadStatus.UPLOADED": "attr = 'UPLOADED'", "skyflow.utils.enums.UpsertType": "class (Enum)", "skyflow.utils.enums.UpsertType.REPLACE": "attr = 'REPLACE'", "skyflow.utils.enums.UpsertType.UPDATE": "attr = 'UPDATE'", "skyflow.vault.controller.VaultController": "class (BaseVaultController)", "skyflow.vault.controller.VaultController.__init__": "def (self, vault_client)", "skyflow.vault.controller.VaultController.delete": "def (self, request: DeleteRequest, options: DeleteOptions = None) -> DeleteResponse", + "skyflow.vault.controller.VaultController.delete_files": "def (self, request: DeleteFilesRequest, options: DeleteFilesOptions = None) -> DeleteFilesResponse", "skyflow.vault.controller.VaultController.detokenize": "def (self, request: DetokenizeRequest, options: DetokenizeOptions = None) -> DetokenizeResponse", "skyflow.vault.controller.VaultController.get": "def (self, request: GetRequest, options: GetOptions = None) -> GetResponse", + "skyflow.vault.controller.VaultController.get_tokens": "def (self, request: GetTokensRequest, options: GetTokensOptions = None) -> GetTokensResponse", "skyflow.vault.controller.VaultController.insert": "def (self, request: InsertRequest, options: InsertOptions = None) -> InsertResponse", + "skyflow.vault.controller.VaultController.query": "def (self, request: QueryRequest, options: QueryOptions = None) -> QueryResponse", "skyflow.vault.controller.VaultController.update": "def (self, request: UpdateRequest, options: UpdateOptions = None) -> UpdateResponse", + "skyflow.vault.controller.VaultController.upload_files": "def (self, request: UploadFilesRequest, options: UploadFilesOptions = None) -> UploadFilesResponse", "skyflow.vault.data.ColumnRedactions": "class ()", "skyflow.vault.data.ColumnRedactions.__init__": "def (self, column_name: str, redaction: str = None)", "skyflow.vault.data.ColumnRedactions.column_name": "attr", @@ -52,6 +60,31 @@ "skyflow.vault.data.CustomHeaderKey.REQUEST_ID_HEADER": "attr = 'x-request-id'", "skyflow.vault.data.CustomHeaderKey.SKYFLOW_ACCOUNT_ID": "attr = 'x-skyflow-account-id'", "skyflow.vault.data.CustomHeaderKey.SKYFLOW_ACCOUNT_NAME": "attr = 'x-skyflow-account-name'", + "skyflow.vault.data.DeleteFilesColumnResult": "class ()", + "skyflow.vault.data.DeleteFilesColumnResult.__init__": "def (self, column: str, status: str = None)", + "skyflow.vault.data.DeleteFilesColumnResult.column": "attr", + "skyflow.vault.data.DeleteFilesColumnResult.status": "attr", + "skyflow.vault.data.DeleteFilesOptions": "class ()", + "skyflow.vault.data.DeleteFilesOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)", + "skyflow.vault.data.DeleteFilesOptions.interceptor": "attr", + "skyflow.vault.data.DeleteFilesRequest": "class (BaseDeleteFilesRequest)", + "skyflow.vault.data.DeleteFilesRequest.__init__": "def (self, records: List[DeleteFilesRequestRecord])", + "skyflow.vault.data.DeleteFilesRequestRecord": "class ()", + "skyflow.vault.data.DeleteFilesRequestRecord.__init__": "def (self, table_name: str, columns: List[str], skyflow_id: str = None, unique_values: List[dict] = None)", + "skyflow.vault.data.DeleteFilesRequestRecord.columns": "attr", + "skyflow.vault.data.DeleteFilesRequestRecord.skyflow_id": "attr", + "skyflow.vault.data.DeleteFilesRequestRecord.table_name": "attr", + "skyflow.vault.data.DeleteFilesRequestRecord.unique_values": "attr", + "skyflow.vault.data.DeleteFilesResponse": "class (BaseDeleteFilesResponse)", + "skyflow.vault.data.DeleteFilesResponse.__init__": "def (self, records = None)", + "skyflow.vault.data.DeleteFilesResponseRecord": "class ()", + "skyflow.vault.data.DeleteFilesResponseRecord.__init__": "def (self, skyflow_id: str = None, table_name: str = None, columns = None, http_code: int = None, error: str = None, request_id: str = None)", + "skyflow.vault.data.DeleteFilesResponseRecord.columns": "attr", + "skyflow.vault.data.DeleteFilesResponseRecord.error": "attr", + "skyflow.vault.data.DeleteFilesResponseRecord.http_code": "attr", + "skyflow.vault.data.DeleteFilesResponseRecord.request_id": "attr", + "skyflow.vault.data.DeleteFilesResponseRecord.skyflow_id": "attr", + "skyflow.vault.data.DeleteFilesResponseRecord.table_name": "attr", "skyflow.vault.data.DeleteOptions": "class ()", "skyflow.vault.data.DeleteOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)", "skyflow.vault.data.DeleteOptions.interceptor": "attr", @@ -125,6 +158,25 @@ "skyflow.vault.data.GetResponseRecord.skyflow_id": "attr", "skyflow.vault.data.GetResponseRecord.table_name": "attr", "skyflow.vault.data.GetResponseRecord.tokens": "attr", + "skyflow.vault.data.GetTokensOptions": "class ()", + "skyflow.vault.data.GetTokensOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)", + "skyflow.vault.data.GetTokensOptions.interceptor": "attr", + "skyflow.vault.data.GetTokensRequest": "class (BaseGetTokensRequest)", + "skyflow.vault.data.GetTokensRequest.__init__": "def (self, records: List[GetTokensRequestRecord])", + "skyflow.vault.data.GetTokensRequestRecord": "class ()", + "skyflow.vault.data.GetTokensRequestRecord.__init__": "def (self, value, token_group_name: str)", + "skyflow.vault.data.GetTokensRequestRecord.token_group_name": "attr", + "skyflow.vault.data.GetTokensRequestRecord.value": "attr", + "skyflow.vault.data.GetTokensResponse": "class (BaseGetTokensResponse)", + "skyflow.vault.data.GetTokensResponse.__init__": "def (self, records = None)", + "skyflow.vault.data.GetTokensResponseRecord": "class ()", + "skyflow.vault.data.GetTokensResponseRecord.__init__": "def (self, value = None, token_group_name = None, token = None, http_code = None, error = None, request_id = None)", + "skyflow.vault.data.GetTokensResponseRecord.error": "attr", + "skyflow.vault.data.GetTokensResponseRecord.http_code": "attr", + "skyflow.vault.data.GetTokensResponseRecord.request_id": "attr", + "skyflow.vault.data.GetTokensResponseRecord.token": "attr", + "skyflow.vault.data.GetTokensResponseRecord.token_group_name": "attr", + "skyflow.vault.data.GetTokensResponseRecord.value": "attr", "skyflow.vault.data.InsertOptions": "class ()", "skyflow.vault.data.InsertOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)", "skyflow.vault.data.InsertOptions.interceptor": "attr", @@ -152,6 +204,21 @@ "skyflow.vault.data.InsertResponseRecord.skyflow_id": "attr", "skyflow.vault.data.InsertResponseRecord.table_name": "attr", "skyflow.vault.data.InsertResponseRecord.tokens": "attr", + "skyflow.vault.data.QueryOptions": "class ()", + "skyflow.vault.data.QueryOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)", + "skyflow.vault.data.QueryOptions.interceptor": "attr", + "skyflow.vault.data.QueryRequest": "class (BaseQueryRequest)", + "skyflow.vault.data.QueryRequest.__init__": "def (self, query: str)", + "skyflow.vault.data.QueryResponse": "class (BaseQueryResponse)", + "skyflow.vault.data.QueryResponse.__init__": "def (self, records = None, metadata = None, request_id = None)", + "skyflow.vault.data.QueryResponse.metadata": "attr", + "skyflow.vault.data.QueryResponse.request_id": "attr", + "skyflow.vault.data.QueryResponseMetadata": "class ()", + "skyflow.vault.data.QueryResponseMetadata.__init__": "def (self, columns: list = None)", + "skyflow.vault.data.QueryResponseMetadata.columns": "attr", + "skyflow.vault.data.QueryResponseRecord": "class ()", + "skyflow.vault.data.QueryResponseRecord.__init__": "def (self, data: dict = None)", + "skyflow.vault.data.QueryResponseRecord.data": "attr", "skyflow.vault.data.RequestContext": "class ()", "skyflow.vault.data.RequestContext.__init__": "def (self, operation, batch_index = NOT_BATCHED, total_batches = NOT_BATCHED)", "skyflow.vault.data.RequestContext.add_header": "def (self, key, value)", @@ -195,6 +262,40 @@ "skyflow.vault.data.UpdateResponseRecord.skyflow_id": "attr", "skyflow.vault.data.UpdateResponseRecord.table_name": "attr", "skyflow.vault.data.UpdateResponseRecord.tokens": "attr", + "skyflow.vault.data.UploadFilesColumnResult": "class ()", + "skyflow.vault.data.UploadFilesColumnResult.__init__": "def (self, column: str, file_name: str = None, upload_status: str = None, error: str = None)", + "skyflow.vault.data.UploadFilesColumnResult.column": "attr", + "skyflow.vault.data.UploadFilesColumnResult.error": "attr", + "skyflow.vault.data.UploadFilesColumnResult.file_name": "attr", + "skyflow.vault.data.UploadFilesColumnResult.upload_status": "attr", + "skyflow.vault.data.UploadFilesOptions": "class ()", + "skyflow.vault.data.UploadFilesOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)", + "skyflow.vault.data.UploadFilesOptions.interceptor": "attr", + "skyflow.vault.data.UploadFilesRequest": "class (BaseUploadFilesRequest)", + "skyflow.vault.data.UploadFilesRequest.__init__": "def (self, records: List[UploadFilesRequestRecord])", + "skyflow.vault.data.UploadFilesRequestColumn": "class ()", + "skyflow.vault.data.UploadFilesRequestColumn.__init__": "def (self, column: str, file_path: str = None, base64: str = None, file_object: BinaryIO = None, file_name: str = None, content_type: str = None)", + "skyflow.vault.data.UploadFilesRequestColumn.base64": "attr", + "skyflow.vault.data.UploadFilesRequestColumn.column": "attr", + "skyflow.vault.data.UploadFilesRequestColumn.content_type": "attr", + "skyflow.vault.data.UploadFilesRequestColumn.file_name": "attr", + "skyflow.vault.data.UploadFilesRequestColumn.file_object": "attr", + "skyflow.vault.data.UploadFilesRequestColumn.file_path": "attr", + "skyflow.vault.data.UploadFilesRequestRecord": "class ()", + "skyflow.vault.data.UploadFilesRequestRecord.__init__": "def (self, table_name: str, columns: List[UploadFilesRequestColumn], skyflow_id: str = None)", + "skyflow.vault.data.UploadFilesRequestRecord.columns": "attr", + "skyflow.vault.data.UploadFilesRequestRecord.skyflow_id": "attr", + "skyflow.vault.data.UploadFilesRequestRecord.table_name": "attr", + "skyflow.vault.data.UploadFilesResponse": "class (BaseUploadFilesResponse)", + "skyflow.vault.data.UploadFilesResponse.__init__": "def (self, records = None)", + "skyflow.vault.data.UploadFilesResponseRecord": "class ()", + "skyflow.vault.data.UploadFilesResponseRecord.__init__": "def (self, skyflow_id: str = None, table_name: str = None, columns = None, http_code: int = None, error: str = None, request_id: str = None)", + "skyflow.vault.data.UploadFilesResponseRecord.columns": "attr", + "skyflow.vault.data.UploadFilesResponseRecord.error": "attr", + "skyflow.vault.data.UploadFilesResponseRecord.http_code": "attr", + "skyflow.vault.data.UploadFilesResponseRecord.request_id": "attr", + "skyflow.vault.data.UploadFilesResponseRecord.skyflow_id": "attr", + "skyflow.vault.data.UploadFilesResponseRecord.table_name": "attr", "skyflow.vault.data.UpsertOptions": "class ()", "skyflow.vault.data.UpsertOptions.__init__": "def (self, unique_columns: list = None, update_type: UpsertType = None)", "skyflow.vault.data.UpsertOptions.unique_columns": "attr", diff --git a/flowvault/samples/README.md b/flowvault/samples/README.md index 5d82e650..a4e1b0dd 100644 --- a/flowvault/samples/README.md +++ b/flowvault/samples/README.md @@ -39,6 +39,10 @@ python flowvault/samples/vault_api/insert_records.py | [update_record.py](vault_api/update_record.py) | Update a record | | [delete_records.py](vault_api/delete_records.py) | Delete records | | [detokenize_records.py](vault_api/detokenize_records.py) | Detokenize tokens | +| [query_records.py](vault_api/query_records.py) | Run a read-only SQL `SELECT` query | +| [get_tokens.py](vault_api/get_tokens.py) | Look up existing tokens for values (deterministic token groups) | +| [upload_files.py](vault_api/upload_files.py) | Upload files to file columns (`file_path` / `base64` / `file_object`) | +| [delete_files.py](vault_api/delete_files.py) | Delete files from file columns (by `skyflow_id` or `unique_values`) | ## Custom headers & HTTP config diff --git a/flowvault/samples/vault_api/delete_files.py b/flowvault/samples/vault_api/delete_files.py new file mode 100644 index 00000000..9c121e87 --- /dev/null +++ b/flowvault/samples/vault_api/delete_files.py @@ -0,0 +1,55 @@ +from skyflow.error import SkyflowError +from skyflow import Env +from skyflow import Skyflow, LogLevel +from skyflow.vault.data import DeleteFilesRequest, DeleteFilesRequestRecord + + +def perform_delete_files(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + # Each record sets exactly ONE of skyflow_id or unique_values to target the record(s). + delete_files_request = DeleteFilesRequest( + records=[ + DeleteFilesRequestRecord( + table_name='', + columns=[''], + skyflow_id='', + ), + ], + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).delete_files(delete_files_request) + + # response.records: list of DeleteFilesResponseRecord, one per resolved record, in order. + # Each has .skyflow_id, .table_name, .columns, .http_code, .error, .request_id. + # Each column result has .column and .status ("DELETED"); .columns is None on a failed record. + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_delete_files() diff --git a/flowvault/samples/vault_api/get_tokens.py b/flowvault/samples/vault_api/get_tokens.py new file mode 100644 index 00000000..949ed0ea --- /dev/null +++ b/flowvault/samples/vault_api/get_tokens.py @@ -0,0 +1,51 @@ +from skyflow.error import SkyflowError +from skyflow import Env +from skyflow import Skyflow, LogLevel +from skyflow.vault.data import GetTokensRequest, GetTokensRequestRecord + + +def perform_get_tokens(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + # One record per input value; only deterministic token groups are supported. + get_tokens_request = GetTokensRequest( + records=[ + GetTokensRequestRecord(value='john@example.com', token_group_name=''), + GetTokensRequestRecord(value='jane@example.com', token_group_name=''), + ], + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).get_tokens(get_tokens_request) + + # response.records: list of GetTokensResponseRecord, one per input, in order. + # Each has .value, .token_group_name, .token, .http_code, .error, .request_id. + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_get_tokens() diff --git a/flowvault/samples/vault_api/query_records.py b/flowvault/samples/vault_api/query_records.py new file mode 100644 index 00000000..bb37c234 --- /dev/null +++ b/flowvault/samples/vault_api/query_records.py @@ -0,0 +1,50 @@ +from skyflow.error import SkyflowError +from skyflow import Env +from skyflow import Skyflow, LogLevel +from skyflow.vault.data import QueryRequest + + +def perform_query(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + # Only SELECT is supported; a call returns at most 25 records -- page with OFFSET. + query_request = QueryRequest( + query="SELECT name, email FROM LIMIT 25", + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).query(query_request) + + # response.records: list of QueryResponseRecord, each with .data (a dict of column -> value). + # response.metadata.columns lists the returned columns; response.request_id is always set. + print('Request ID: ', response.request_id) + print('Columns: ', response.metadata.columns if response.metadata else None) + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_query() diff --git a/flowvault/samples/vault_api/upload_files.py b/flowvault/samples/vault_api/upload_files.py new file mode 100644 index 00000000..8702c0a1 --- /dev/null +++ b/flowvault/samples/vault_api/upload_files.py @@ -0,0 +1,57 @@ +from skyflow.error import SkyflowError +from skyflow import Env +from skyflow import Skyflow, LogLevel +from skyflow.vault.data import UploadFilesRequest, UploadFilesRequestRecord, UploadFilesRequestColumn + + +def perform_upload_files(): + try: + credentials = { + 'path': '', + } + + vault_config = { + 'vault_id': '', + 'cluster_id': '', + 'env': Env.PROD, + 'credentials': credentials, + } + + skyflow_client = ( + Skyflow.builder() + .add_vault_config(vault_config) + .set_log_level(LogLevel.ERROR) + .build() + ) + + # Each column takes exactly ONE of file_path / base64 (needs file_name) / file_object. + # Omit skyflow_id to create a new record; set it to upload into an existing record. + upload_files_request = UploadFilesRequest( + records=[ + UploadFilesRequestRecord( + table_name='', + columns=[ + UploadFilesRequestColumn(column='', file_path=''), + ], + ), + ], + ) + + response = skyflow_client.vault(vault_config.get('vault_id')).upload_files(upload_files_request) + + # response.records: list of UploadFilesResponseRecord, one per input record, in order. + # Each has .skyflow_id, .table_name, .columns, .http_code, .error, .request_id. + # Each column result has .column, .file_name, .upload_status ("UPLOADED"/"FAILED"/"SKIPPED"), .error. + print('Records: ', response.records) + + except SkyflowError as error: + print('Skyflow Specific Error: ', { + 'code': error.http_code, + 'message': error.message, + 'details': error.details, + }) + except Exception as error: + print('Unexpected Error:', error) + + +perform_upload_files() diff --git a/flowvault/skyflow/generated/rest/__init__.py b/flowvault/skyflow/generated/rest/__init__.py index ad0c973d..60843228 100644 --- a/flowvault/skyflow/generated/rest/__init__.py +++ b/flowvault/skyflow/generated/rest/__init__.py @@ -17,6 +17,13 @@ ExecuteQueryRecordResponse, ExecuteQueryResponse, ExecuteQueryResponseMetadata, + FileDeleteRecord, + FileDeleteResponse, + FileDeleteResponseObject, + FileUploadColumn, + FileUploadRecord, + FileUploadResponse, + FileUploadResponseObject, GetRequestData, GetResponse, GetTokensFromValuesRequestObject, @@ -43,7 +50,7 @@ TooManyRequestsError, UnauthorizedError, ) - from . import query, records, tokens + from . import files, query, records, tokens from ._default_clients import DefaultAioHttpClient, DefaultAsyncHttpxClient from .client import AsyncSkyflowAuth, SkyflowAuth from .environment import SkyflowAuthEnvironment @@ -64,6 +71,13 @@ "ExecuteQueryRecordResponse": ".types", "ExecuteQueryResponse": ".types", "ExecuteQueryResponseMetadata": ".types", + "FileDeleteRecord": ".types", + "FileDeleteResponse": ".types", + "FileDeleteResponseObject": ".types", + "FileUploadColumn": ".types", + "FileUploadRecord": ".types", + "FileUploadResponse": ".types", + "FileUploadResponseObject": ".types", "ForbiddenError": ".errors", "GetRequestData": ".types", "GetResponse": ".types", @@ -90,6 +104,7 @@ "Upsert": ".types", "UpsertUpdateType": ".types", "__version__": ".version", + "files": ".files", "query": ".query", "records": ".records", "tokens": ".tokens", @@ -132,6 +147,13 @@ def __dir__(): "ExecuteQueryRecordResponse", "ExecuteQueryResponse", "ExecuteQueryResponseMetadata", + "FileDeleteRecord", + "FileDeleteResponse", + "FileDeleteResponseObject", + "FileUploadColumn", + "FileUploadRecord", + "FileUploadResponse", + "FileUploadResponseObject", "ForbiddenError", "GetRequestData", "GetResponse", @@ -158,6 +180,7 @@ def __dir__(): "Upsert", "UpsertUpdateType", "__version__", + "files", "query", "records", "tokens", diff --git a/flowvault/skyflow/generated/rest/client.py b/flowvault/skyflow/generated/rest/client.py index c100c8dd..1f50d133 100644 --- a/flowvault/skyflow/generated/rest/client.py +++ b/flowvault/skyflow/generated/rest/client.py @@ -12,6 +12,7 @@ from .raw_client import AsyncRawSkyflowAuth, RawSkyflowAuth if typing.TYPE_CHECKING: + from .files.client import AsyncFilesClient, FilesClient from .query.client import AsyncQueryClient, QueryClient from .records.client import AsyncRecordsClient, RecordsClient from .tokens.client import AsyncTokensClient, TokensClient @@ -115,6 +116,7 @@ def __init__( logging=logging, ) self._raw_client = RawSkyflowAuth(client_wrapper=self._client_wrapper) + self._files: typing.Optional[FilesClient] = None self._query: typing.Optional[QueryClient] = None self._records: typing.Optional[RecordsClient] = None self._tokens: typing.Optional[TokensClient] = None @@ -157,6 +159,14 @@ def patch_v2vaults_id( _response = self._raw_client.patch_v2vaults_id(vault_id, request_options=request_options) return _response.data + @property + def files(self): + if self._files is None: + from .files.client import FilesClient # noqa: E402 + + self._files = FilesClient(client_wrapper=self._client_wrapper) + return self._files + @property def query(self): if self._query is None: @@ -301,6 +311,7 @@ def __init__( logging=logging, ) self._raw_client = AsyncRawSkyflowAuth(client_wrapper=self._client_wrapper) + self._files: typing.Optional[AsyncFilesClient] = None self._query: typing.Optional[AsyncQueryClient] = None self._records: typing.Optional[AsyncRecordsClient] = None self._tokens: typing.Optional[AsyncTokensClient] = None @@ -351,6 +362,14 @@ async def main() -> None: _response = await self._raw_client.patch_v2vaults_id(vault_id, request_options=request_options) return _response.data + @property + def files(self): + if self._files is None: + from .files.client import AsyncFilesClient # noqa: E402 + + self._files = AsyncFilesClient(client_wrapper=self._client_wrapper) + return self._files + @property def query(self): if self._query is None: diff --git a/flowvault/skyflow/generated/rest/core/client_wrapper.py b/flowvault/skyflow/generated/rest/core/client_wrapper.py index edb7b114..a8ad5288 100644 --- a/flowvault/skyflow/generated/rest/core/client_wrapper.py +++ b/flowvault/skyflow/generated/rest/core/client_wrapper.py @@ -37,7 +37,7 @@ def get_headers(self) -> typing.Dict[str, str]: "X-Fern-Runtime": f"python/{platform.python_version()}", "X-Fern-Platform": f"{platform.system().lower()}/{platform.release()}", "X-Fern-SDK-Name": "skyflow.generated.rest", - "X-Fern-SDK-Version": "0.0.23", + "X-Fern-SDK-Version": "0.0.27", **(self.get_custom_headers() or {}), } token = self._get_token() diff --git a/flowvault/skyflow/generated/rest/files/__init__.py b/flowvault/skyflow/generated/rest/files/__init__.py new file mode 100644 index 00000000..5cde0202 --- /dev/null +++ b/flowvault/skyflow/generated/rest/files/__init__.py @@ -0,0 +1,4 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + diff --git a/flowvault/skyflow/generated/rest/files/client.py b/flowvault/skyflow/generated/rest/files/client.py new file mode 100644 index 00000000..7a050e06 --- /dev/null +++ b/flowvault/skyflow/generated/rest/files/client.py @@ -0,0 +1,270 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.request_options import RequestOptions +from ..types.file_delete_record import FileDeleteRecord +from ..types.file_delete_response import FileDeleteResponse +from ..types.file_upload_record import FileUploadRecord +from ..types.file_upload_response import FileUploadResponse +from .raw_client import AsyncRawFilesClient, RawFilesClient + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class FilesClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._raw_client = RawFilesClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> RawFilesClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + RawFilesClient + """ + return self._raw_client + + def delete_files( + self, + *, + vault_id: str, + records: typing.Sequence[FileDeleteRecord], + request_options: typing.Optional[RequestOptions] = None, + ) -> FileDeleteResponse: + """ + Deletes one or more files from the specified file column(s) of existing record(s) in a vault, and removes the underlying object from cloud storage. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[FileDeleteRecord] + List of record objects (max 20,000) identifying the file columns to delete. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + FileDeleteResponse + OK + + Examples + -------- + from skyflow import FileDeleteRecord, SkyflowAuth + + client = SkyflowAuth( + token="YOUR_TOKEN", + ) + client.files.delete_files( + vault_id="d408485953784308a000f8dcf81901ef", + records=[ + FileDeleteRecord( + table_name="onboarding", + skyflow_id="77ea0577-894a-4a42-90a9-51b0eaf97159", + columns=["resumePDF", "photoID"], + ) + ], + ) + """ + _response = self._raw_client.delete_files(vault_id=vault_id, records=records, request_options=request_options) + return _response.data + + def upload_files( + self, + *, + vault_id: str, + records: typing.Sequence[FileUploadRecord], + request_options: typing.Optional[RequestOptions] = None, + ) -> FileUploadResponse: + """ + Generates signed URLs for uploading files to specified columns in a vault. Returns a signed URL per column that expires in 15 minutes by default. If a skyflowID is not provided for a record, a new record is created and its skyflowID is returned. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[FileUploadRecord] + List of records for which signed file upload URLs are requested. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + FileUploadResponse + OK + + Examples + -------- + from skyflow import FileUploadColumn, FileUploadRecord, SkyflowAuth + + client = SkyflowAuth( + token="YOUR_TOKEN", + ) + client.files.upload_files( + vault_id="d408485953784308a000f8dcf81901ef", + records=[ + FileUploadRecord( + table_name="onboarding", + columns=[ + FileUploadColumn( + column="resumePDF", + file_name="sampleresume.pdf", + ), + FileUploadColumn( + column="photoID", + file_name="samplephoto.jpg", + ), + ], + ) + ], + ) + """ + _response = self._raw_client.upload_files(vault_id=vault_id, records=records, request_options=request_options) + return _response.data + + +class AsyncFilesClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._raw_client = AsyncRawFilesClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> AsyncRawFilesClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + AsyncRawFilesClient + """ + return self._raw_client + + async def delete_files( + self, + *, + vault_id: str, + records: typing.Sequence[FileDeleteRecord], + request_options: typing.Optional[RequestOptions] = None, + ) -> FileDeleteResponse: + """ + Deletes one or more files from the specified file column(s) of existing record(s) in a vault, and removes the underlying object from cloud storage. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[FileDeleteRecord] + List of record objects (max 20,000) identifying the file columns to delete. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + FileDeleteResponse + OK + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflowAuth, FileDeleteRecord + + client = AsyncSkyflowAuth( + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.files.delete_files( + vault_id="d408485953784308a000f8dcf81901ef", + records=[ + FileDeleteRecord( + table_name="onboarding", + skyflow_id="77ea0577-894a-4a42-90a9-51b0eaf97159", + columns=["resumePDF", "photoID"], + ) + ], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.delete_files( + vault_id=vault_id, records=records, request_options=request_options + ) + return _response.data + + async def upload_files( + self, + *, + vault_id: str, + records: typing.Sequence[FileUploadRecord], + request_options: typing.Optional[RequestOptions] = None, + ) -> FileUploadResponse: + """ + Generates signed URLs for uploading files to specified columns in a vault. Returns a signed URL per column that expires in 15 minutes by default. If a skyflowID is not provided for a record, a new record is created and its skyflowID is returned. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[FileUploadRecord] + List of records for which signed file upload URLs are requested. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + FileUploadResponse + OK + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflowAuth, FileUploadColumn, FileUploadRecord + + client = AsyncSkyflowAuth( + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.files.upload_files( + vault_id="d408485953784308a000f8dcf81901ef", + records=[ + FileUploadRecord( + table_name="onboarding", + columns=[ + FileUploadColumn( + column="resumePDF", + file_name="sampleresume.pdf", + ), + FileUploadColumn( + column="photoID", + file_name="samplephoto.jpg", + ), + ], + ) + ], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.upload_files( + vault_id=vault_id, records=records, request_options=request_options + ) + return _response.data diff --git a/flowvault/skyflow/generated/rest/files/raw_client.py b/flowvault/skyflow/generated/rest/files/raw_client.py new file mode 100644 index 00000000..44420695 --- /dev/null +++ b/flowvault/skyflow/generated/rest/files/raw_client.py @@ -0,0 +1,541 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from json.decoder import JSONDecodeError + +from ..core.api_error import ApiError +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.http_response import AsyncHttpResponse, HttpResponse +from ..core.parse_error import ParsingError +from ..core.pydantic_utilities import parse_obj_as +from ..core.request_options import RequestOptions +from ..core.serialization import convert_and_respect_annotation_metadata +from ..errors.bad_request_error import BadRequestError +from ..errors.forbidden_error import ForbiddenError +from ..errors.internal_server_error import InternalServerError +from ..errors.not_found_error import NotFoundError +from ..errors.too_many_requests_error import TooManyRequestsError +from ..errors.unauthorized_error import UnauthorizedError +from ..types.error_response import ErrorResponse +from ..types.file_delete_record import FileDeleteRecord +from ..types.file_delete_response import FileDeleteResponse +from ..types.file_upload_record import FileUploadRecord +from ..types.file_upload_response import FileUploadResponse +from pydantic import ValidationError + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class RawFilesClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._client_wrapper = client_wrapper + + def delete_files( + self, + *, + vault_id: str, + records: typing.Sequence[FileDeleteRecord], + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[FileDeleteResponse]: + """ + Deletes one or more files from the specified file column(s) of existing record(s) in a vault, and removes the underlying object from cloud storage. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[FileDeleteRecord] + List of record objects (max 20,000) identifying the file columns to delete. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[FileDeleteResponse] + OK + """ + _response = self._client_wrapper.httpx_client.request( + "v2/files/delete", + method="POST", + json={ + "vaultID": vault_id, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[FileDeleteRecord], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + FileDeleteResponse, + parse_obj_as( + type_=FileDeleteResponse, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise ForbiddenError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + except ValidationError as e: + raise ParsingError( + status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e + ) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + def upload_files( + self, + *, + vault_id: str, + records: typing.Sequence[FileUploadRecord], + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[FileUploadResponse]: + """ + Generates signed URLs for uploading files to specified columns in a vault. Returns a signed URL per column that expires in 15 minutes by default. If a skyflowID is not provided for a record, a new record is created and its skyflowID is returned. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[FileUploadRecord] + List of records for which signed file upload URLs are requested. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[FileUploadResponse] + OK + """ + _response = self._client_wrapper.httpx_client.request( + "v2/files/upload", + method="POST", + json={ + "vaultID": vault_id, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[FileUploadRecord], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + FileUploadResponse, + parse_obj_as( + type_=FileUploadResponse, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise ForbiddenError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + except ValidationError as e: + raise ParsingError( + status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e + ) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + +class AsyncRawFilesClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def delete_files( + self, + *, + vault_id: str, + records: typing.Sequence[FileDeleteRecord], + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[FileDeleteResponse]: + """ + Deletes one or more files from the specified file column(s) of existing record(s) in a vault, and removes the underlying object from cloud storage. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[FileDeleteRecord] + List of record objects (max 20,000) identifying the file columns to delete. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[FileDeleteResponse] + OK + """ + _response = await self._client_wrapper.httpx_client.request( + "v2/files/delete", + method="POST", + json={ + "vaultID": vault_id, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[FileDeleteRecord], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + FileDeleteResponse, + parse_obj_as( + type_=FileDeleteResponse, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise ForbiddenError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + except ValidationError as e: + raise ParsingError( + status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e + ) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def upload_files( + self, + *, + vault_id: str, + records: typing.Sequence[FileUploadRecord], + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[FileUploadResponse]: + """ + Generates signed URLs for uploading files to specified columns in a vault. Returns a signed URL per column that expires in 15 minutes by default. If a skyflowID is not provided for a record, a new record is created and its skyflowID is returned. + + Parameters + ---------- + vault_id : str + ID of the vault. + + records : typing.Sequence[FileUploadRecord] + List of records for which signed file upload URLs are requested. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[FileUploadResponse] + OK + """ + _response = await self._client_wrapper.httpx_client.request( + "v2/files/upload", + method="POST", + json={ + "vaultID": vault_id, + "records": convert_and_respect_annotation_metadata( + object_=records, annotation=typing.Sequence[FileUploadRecord], direction="write" + ), + }, + headers={ + "content-type": "application/json", + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + FileUploadResponse, + parse_obj_as( + type_=FileUploadResponse, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise ForbiddenError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + except ValidationError as e: + raise ParsingError( + status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e + ) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) diff --git a/flowvault/skyflow/generated/rest/types/__init__.py b/flowvault/skyflow/generated/rest/types/__init__.py index 28e97995..cdc8431f 100644 --- a/flowvault/skyflow/generated/rest/types/__init__.py +++ b/flowvault/skyflow/generated/rest/types/__init__.py @@ -16,6 +16,13 @@ from .execute_query_record_response import ExecuteQueryRecordResponse from .execute_query_response import ExecuteQueryResponse from .execute_query_response_metadata import ExecuteQueryResponseMetadata + from .file_delete_record import FileDeleteRecord + from .file_delete_response import FileDeleteResponse + from .file_delete_response_object import FileDeleteResponseObject + from .file_upload_column import FileUploadColumn + from .file_upload_record import FileUploadRecord + from .file_upload_response import FileUploadResponse + from .file_upload_response_object import FileUploadResponseObject from .get_request_data import GetRequestData from .get_response import GetResponse from .get_tokens_from_values_request_object import GetTokensFromValuesRequestObject @@ -44,6 +51,13 @@ "ExecuteQueryRecordResponse": ".execute_query_record_response", "ExecuteQueryResponse": ".execute_query_response", "ExecuteQueryResponseMetadata": ".execute_query_response_metadata", + "FileDeleteRecord": ".file_delete_record", + "FileDeleteResponse": ".file_delete_response", + "FileDeleteResponseObject": ".file_delete_response_object", + "FileUploadColumn": ".file_upload_column", + "FileUploadRecord": ".file_upload_record", + "FileUploadResponse": ".file_upload_response", + "FileUploadResponseObject": ".file_upload_response_object", "GetRequestData": ".get_request_data", "GetResponse": ".get_response", "GetTokensFromValuesRequestObject": ".get_tokens_from_values_request_object", @@ -96,6 +110,13 @@ def __dir__(): "ExecuteQueryRecordResponse", "ExecuteQueryResponse", "ExecuteQueryResponseMetadata", + "FileDeleteRecord", + "FileDeleteResponse", + "FileDeleteResponseObject", + "FileUploadColumn", + "FileUploadRecord", + "FileUploadResponse", + "FileUploadResponseObject", "GetRequestData", "GetResponse", "GetTokensFromValuesRequestObject", diff --git a/flowvault/skyflow/generated/rest/types/file_delete_record.py b/flowvault/skyflow/generated/rest/types/file_delete_record.py new file mode 100644 index 00000000..b0eebe24 --- /dev/null +++ b/flowvault/skyflow/generated/rest/types/file_delete_record.py @@ -0,0 +1,56 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from ..core.serialization import FieldMetadata +from .unique_value import UniqueValue + + +class FileDeleteRecord(UniversalBaseModel): + table_name: typing_extensions.Annotated[ + str, FieldMetadata(alias="tableName"), pydantic.Field(alias="tableName", description="Name of the table.") + ] + """ + Name of the table. + """ + + skyflow_id: typing_extensions.Annotated[ + typing.Optional[str], + FieldMetadata(alias="skyflowID"), + pydantic.Field( + alias="skyflowID", + description="Skyflow ID of the record containing the file column(s) to delete. Either skyflowID or uniqueValues must be specified, not both.", + ), + ] = None + """ + Skyflow ID of the record containing the file column(s) to delete. Either skyflowID or uniqueValues must be specified, not both. + """ + + columns: typing.List[str] = pydantic.Field() + """ + Names of the file columns to delete on this record. If any listed column has no file present, the request fails for this record. + """ + + unique_values: typing_extensions.Annotated[ + typing.Optional[typing.List[UniqueValue]], + FieldMetadata(alias="uniqueValues"), + pydantic.Field( + alias="uniqueValues", + description="List of unique constraint values used to look up the record(s) to delete, in place of skyflowID. If more than one object is provided, each is resolved independently and the specified columns are deleted from every matching record.", + ), + ] = None + """ + List of unique constraint values used to look up the record(s) to delete, in place of skyflowID. If more than one object is provided, each is resolved independently and the specified columns are deleted from every matching record. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow/generated/rest/types/file_delete_response.py b/flowvault/skyflow/generated/rest/types/file_delete_response.py new file mode 100644 index 00000000..732ba60e --- /dev/null +++ b/flowvault/skyflow/generated/rest/types/file_delete_response.py @@ -0,0 +1,23 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from .file_delete_response_object import FileDeleteResponseObject + + +class FileDeleteResponse(UniversalBaseModel): + records: typing.List[FileDeleteResponseObject] = pydantic.Field() + """ + List of records (max 100) with the result of the delete operation. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow/generated/rest/types/file_delete_response_object.py b/flowvault/skyflow/generated/rest/types/file_delete_response_object.py new file mode 100644 index 00000000..35ff7e46 --- /dev/null +++ b/flowvault/skyflow/generated/rest/types/file_delete_response_object.py @@ -0,0 +1,59 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from ..core.serialization import FieldMetadata + + +class FileDeleteResponseObject(UniversalBaseModel): + skyflow_id: typing_extensions.Annotated[ + typing.Optional[str], + FieldMetadata(alias="skyflowID"), + pydantic.Field( + alias="skyflowID", + description="Skyflow ID of the record that was targeted. When the request specifies uniqueValues, this is the skyflowID resolved from the matching record. Null if the record failed validation before a skyflowID could be resolved.", + ), + ] = None + """ + Skyflow ID of the record that was targeted. When the request specifies uniqueValues, this is the skyflowID resolved from the matching record. Null if the record failed validation before a skyflowID could be resolved. + """ + + table_name: typing_extensions.Annotated[ + str, FieldMetadata(alias="tableName"), pydantic.Field(alias="tableName", description="Name of the table.") + ] + """ + Name of the table. + """ + + data: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field(default=None) + """ + Map of column name to deletion result. Populated only when the record's delete succeeds, with one entry per requested column with status DELETED. Null if any requested column has no file present, in which case error is populated instead. + """ + + error: typing.Optional[str] = pydantic.Field(default=None) + """ + Error message describing any issue with the specific record. Null if no error occurred. + """ + + http_code: typing_extensions.Annotated[ + int, + FieldMetadata(alias="httpCode"), + pydantic.Field( + alias="httpCode", description="HTTP status code representing the result of the API request for this record." + ), + ] + """ + HTTP status code representing the result of the API request for this record. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow/generated/rest/types/file_upload_column.py b/flowvault/skyflow/generated/rest/types/file_upload_column.py new file mode 100644 index 00000000..e937bcac --- /dev/null +++ b/flowvault/skyflow/generated/rest/types/file_upload_column.py @@ -0,0 +1,36 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from ..core.serialization import FieldMetadata + + +class FileUploadColumn(UniversalBaseModel): + column: str = pydantic.Field() + """ + Name of the column where the file is being uploaded. + """ + + file_name: typing_extensions.Annotated[ + typing.Optional[str], + FieldMetadata(alias="fileName"), + pydantic.Field( + alias="fileName", + description="Name of the file being uploaded to the file column. If not provided, a random 16-byte UUID name is generated.", + ), + ] = None + """ + Name of the file being uploaded to the file column. If not provided, a random 16-byte UUID name is generated. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow/generated/rest/types/file_upload_record.py b/flowvault/skyflow/generated/rest/types/file_upload_record.py new file mode 100644 index 00000000..a38bf8a0 --- /dev/null +++ b/flowvault/skyflow/generated/rest/types/file_upload_record.py @@ -0,0 +1,44 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from ..core.serialization import FieldMetadata +from .file_upload_column import FileUploadColumn + + +class FileUploadRecord(UniversalBaseModel): + table_name: typing_extensions.Annotated[ + str, FieldMetadata(alias="tableName"), pydantic.Field(alias="tableName", description="Name of the table.") + ] + """ + Name of the table. + """ + + skyflow_id: typing_extensions.Annotated[ + typing.Optional[str], + FieldMetadata(alias="skyflowID"), + pydantic.Field( + alias="skyflowID", + description="Skyflow ID of the record. If not provided, a new record is created and its skyflowID is used for the file upload.", + ), + ] = None + """ + Skyflow ID of the record. If not provided, a new record is created and its skyflowID is used for the file upload. + """ + + columns: typing.List[FileUploadColumn] = pydantic.Field() + """ + List of file columns to generate signed upload URLs for. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow/generated/rest/types/file_upload_response.py b/flowvault/skyflow/generated/rest/types/file_upload_response.py new file mode 100644 index 00000000..ef2c7e61 --- /dev/null +++ b/flowvault/skyflow/generated/rest/types/file_upload_response.py @@ -0,0 +1,23 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from .file_upload_response_object import FileUploadResponseObject + + +class FileUploadResponse(UniversalBaseModel): + records: typing.List[FileUploadResponseObject] = pydantic.Field() + """ + List of records with signed upload URLs. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow/generated/rest/types/file_upload_response_object.py b/flowvault/skyflow/generated/rest/types/file_upload_response_object.py new file mode 100644 index 00000000..5654b289 --- /dev/null +++ b/flowvault/skyflow/generated/rest/types/file_upload_response_object.py @@ -0,0 +1,59 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from ..core.serialization import FieldMetadata + + +class FileUploadResponseObject(UniversalBaseModel): + skyflow_id: typing_extensions.Annotated[ + str, + FieldMetadata(alias="skyflowID"), + pydantic.Field( + alias="skyflowID", + description="Skyflow ID of the record. A new skyflowID is returned if one was not provided in the request.", + ), + ] + """ + Skyflow ID of the record. A new skyflowID is returned if one was not provided in the request. + """ + + table_name: typing_extensions.Annotated[ + str, FieldMetadata(alias="tableName"), pydantic.Field(alias="tableName", description="Name of the table.") + ] + """ + Name of the table. + """ + + data: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field(default=None) + """ + Map of column names to signed upload URLs. Each signed URL expires after 15 minutes by default. + """ + + error: typing.Optional[str] = pydantic.Field(default=None) + """ + Error message describing any issue with the specific record. Null if no error occurred. + """ + + http_code: typing_extensions.Annotated[ + int, + FieldMetadata(alias="httpCode"), + pydantic.Field( + alias="httpCode", description="HTTP status code representing the result of the API request for this record." + ), + ] + """ + HTTP status code representing the result of the API request for this record. + """ + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow diff --git a/flowvault/skyflow/utils/_skyflow_messages.py b/flowvault/skyflow/utils/_skyflow_messages.py index a5f511c6..7c3e720a 100644 --- a/flowvault/skyflow/utils/_skyflow_messages.py +++ b/flowvault/skyflow/utils/_skyflow_messages.py @@ -80,6 +80,27 @@ class Error(Enum): INVALID_TOKENS_TYPE_IN_DETOKENIZE = f"{error_prefix} Detokenize failed. 'tokens' must be a non-empty list of strings." INVALID_TOKEN_GROUP_REDACTIONS_IN_DETOKENIZE = f"{error_prefix} Detokenize failed. 'token_group_redactions' must be a list of TokenGroupRedactions objects with a non-empty 'token_group_name'." + INVALID_QUERY_IN_QUERY = f"{error_prefix} Query failed. 'query' must be a non-empty string." + + INVALID_RECORDS_TYPE_IN_GET_TOKENS = f"{error_prefix} Get tokens failed. 'records' must be a non-empty list of GetTokensRequestRecord objects." + MISSING_VALUE_IN_GET_TOKENS = f"{error_prefix} Get tokens failed. Each record must specify a non-null 'value'." + MISSING_TOKEN_GROUP_NAME_IN_GET_TOKENS = f"{error_prefix} Get tokens failed. Each record must specify a non-empty 'token_group_name'." + + INVALID_RECORDS_TYPE_IN_UPLOAD_FILES = f"{error_prefix} Upload files failed. 'records' must be a non-empty list of UploadFilesRequestRecord objects." + MISSING_TABLE_NAME_IN_UPLOAD_FILES = f"{error_prefix} Upload files failed. Each record must specify a non-empty 'table_name'." + INVALID_COLUMNS_TYPE_IN_UPLOAD_FILES = f"{error_prefix} Upload files failed. Each record's 'columns' must be a non-empty list of UploadFilesRequestColumn objects." + MISSING_COLUMN_NAME_IN_UPLOAD_FILES = f"{error_prefix} Upload files failed. Each column must specify a non-empty 'column'." + MISSING_FILE_SOURCE_IN_UPLOAD_FILES = f"{error_prefix} Upload files failed. Each column must specify exactly one of 'file_path', 'base64' or 'file_object'." + MULTIPLE_FILE_SOURCES_IN_UPLOAD_FILES = f"{error_prefix} Upload files failed. Provide only one of 'file_path', 'base64' or 'file_object' per column." + MISSING_FILE_NAME_FOR_BASE64_IN_UPLOAD_FILES = f"{error_prefix} Upload files failed. 'file_name' is required when 'base64' is used." + FILE_NOT_FOUND_IN_UPLOAD_FILES = f"{error_prefix} Upload files failed. Could not read file at path '{{}}'." + INVALID_BASE64_IN_UPLOAD_FILES = f"{error_prefix} Upload files failed. 'base64' content for column '{{}}' could not be decoded." + + INVALID_RECORDS_TYPE_IN_DELETE_FILES = f"{error_prefix} Delete files failed. 'records' must be a non-empty list of DeleteFilesRequestRecord objects." + MISSING_TABLE_NAME_IN_DELETE_FILES = f"{error_prefix} Delete files failed. Each record must specify a non-empty 'table_name'." + INVALID_COLUMNS_IN_DELETE_FILES = f"{error_prefix} Delete files failed. Each record's 'columns' must be a non-empty list of strings." + INVALID_ID_OR_UNIQUE_VALUES_IN_DELETE_FILES = f"{error_prefix} Delete files failed. Set exactly one of 'skyflow_id' or 'unique_values' per record." + class Info(Enum): VALIDATE_INSERT_REQUEST = f"{INFO}: [{error_prefix}] Validating insert request." INSERT_TRIGGERED = f"{INFO}: [{error_prefix}] Insert method triggered." @@ -106,9 +127,33 @@ class Info(Enum): DETOKENIZE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Detokenize request resolved." DETOKENIZE_SUCCESS = f"{INFO}: [{error_prefix}] Tokens detokenized." + VALIDATE_QUERY_REQUEST = f"{INFO}: [{error_prefix}] Validating query request." + QUERY_TRIGGERED = f"{INFO}: [{error_prefix}] Query method triggered." + QUERY_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Query request resolved." + QUERY_SUCCESS = f"{INFO}: [{error_prefix}] Query executed." + + VALIDATE_GET_TOKENS_REQUEST = f"{INFO}: [{error_prefix}] Validating get tokens request." + GET_TOKENS_TRIGGERED = f"{INFO}: [{error_prefix}] Get tokens method triggered." + GET_TOKENS_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Get tokens request resolved." + GET_TOKENS_SUCCESS = f"{INFO}: [{error_prefix}] Tokens fetched." + + VALIDATE_UPLOAD_FILES_REQUEST = f"{INFO}: [{error_prefix}] Validating upload files request." + UPLOAD_FILES_TRIGGERED = f"{INFO}: [{error_prefix}] Upload files method triggered." + UPLOAD_FILES_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Upload files request resolved." + UPLOAD_FILES_SUCCESS = f"{INFO}: [{error_prefix}] Files uploaded." + + VALIDATE_DELETE_FILES_REQUEST = f"{INFO}: [{error_prefix}] Validating delete files request." + DELETE_FILES_TRIGGERED = f"{INFO}: [{error_prefix}] Delete files method triggered." + DELETE_FILES_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Delete files request resolved." + DELETE_FILES_SUCCESS = f"{INFO}: [{error_prefix}] Files deleted." + class ErrorLogs(Enum): INSERT_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Insert call resulted in failure." GET_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Get call resulted in failure." UPDATE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Update call resulted in failure." DELETE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Delete call resulted in failure." DETOKENIZE_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Detokenize call resulted in failure." + QUERY_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Query call resulted in failure." + GET_TOKENS_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Get tokens call resulted in failure." + UPLOAD_FILES_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Upload files call resulted in failure." + DELETE_FILES_RECORDS_REJECTED = f"{ERROR}: [{error_prefix}] Delete files call resulted in failure." diff --git a/flowvault/skyflow/utils/enums/__init__.py b/flowvault/skyflow/utils/enums/__init__.py index d9dac7e5..8bd9d9d4 100644 --- a/flowvault/skyflow/utils/enums/__init__.py +++ b/flowvault/skyflow/utils/enums/__init__.py @@ -1,3 +1,4 @@ from ._upsert_type import UpsertType from ._env_urls import EnvUrls from ._custom_header_key import CustomHeaderKey +from ._file_upload_status import FileUploadStatus diff --git a/flowvault/skyflow/utils/enums/_file_upload_status.py b/flowvault/skyflow/utils/enums/_file_upload_status.py new file mode 100644 index 00000000..31643443 --- /dev/null +++ b/flowvault/skyflow/utils/enums/_file_upload_status.py @@ -0,0 +1,7 @@ +from enum import Enum + + +class FileUploadStatus(Enum): + UPLOADED = "UPLOADED" + FAILED = "FAILED" + SKIPPED = "SKIPPED" diff --git a/flowvault/skyflow/utils/validations/__init__.py b/flowvault/skyflow/utils/validations/__init__.py index b37fba7e..f4104fcd 100644 --- a/flowvault/skyflow/utils/validations/__init__.py +++ b/flowvault/skyflow/utils/validations/__init__.py @@ -6,4 +6,8 @@ validate_update_request, validate_delete_request, validate_detokenize_request, + validate_query_request, + validate_get_tokens_request, + validate_upload_files_request, + validate_delete_files_request, ) diff --git a/flowvault/skyflow/utils/validations/_validations.py b/flowvault/skyflow/utils/validations/_validations.py index 06727f52..7dc00b17 100644 --- a/flowvault/skyflow/utils/validations/_validations.py +++ b/flowvault/skyflow/utils/validations/_validations.py @@ -15,7 +15,17 @@ VAULT_URL_KEY, VAULT_CONFIG_KEYS, ) -from skyflow.vault.data import GetRequestRecord, InsertRequestRecord, UpdateRequestRecord, UpsertOptions, TokenGroupRedactions +from skyflow.vault.data import ( + GetRequestRecord, + InsertRequestRecord, + UpdateRequestRecord, + UpsertOptions, + TokenGroupRedactions, + GetTokensRequestRecord, + UploadFilesRequestRecord, + UploadFilesRequestColumn, + DeleteFilesRequestRecord, +) VALID_UPDATE_RECORD_KEYS = ["skyflow_id", "data", "tokens", "table_name"] @@ -193,3 +203,61 @@ def validate_detokenize_request(logger, request): ) if not valid: raise SkyflowError(SkyflowMessages.Error.INVALID_TOKEN_GROUP_REDACTIONS_IN_DETOKENIZE.value, invalid_input_error_code) + + +def validate_query_request(logger, request): + if not isinstance(request.query, str) or not request.query.strip(): + raise SkyflowError(SkyflowMessages.Error.INVALID_QUERY_IN_QUERY.value, invalid_input_error_code) + + +def validate_get_tokens_request(logger, request): + if (not isinstance(request.records, list) or not request.records + or not all(isinstance(r, GetTokensRequestRecord) for r in request.records)): + raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_GET_TOKENS.value, invalid_input_error_code) + + for record in request.records: + if record.value is None: + raise SkyflowError(SkyflowMessages.Error.MISSING_VALUE_IN_GET_TOKENS.value, invalid_input_error_code) + if not isinstance(record.token_group_name, str) or not record.token_group_name.strip(): + raise SkyflowError(SkyflowMessages.Error.MISSING_TOKEN_GROUP_NAME_IN_GET_TOKENS.value, invalid_input_error_code) + + +def validate_upload_files_request(logger, request): + if (not isinstance(request.records, list) or not request.records + or not all(isinstance(r, UploadFilesRequestRecord) for r in request.records)): + raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_UPLOAD_FILES.value, invalid_input_error_code) + + for record in request.records: + if not isinstance(record.table_name, str) or not record.table_name.strip(): + raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_UPLOAD_FILES.value, invalid_input_error_code) + if (not isinstance(record.columns, list) or not record.columns + or not all(isinstance(c, UploadFilesRequestColumn) for c in record.columns)): + raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMNS_TYPE_IN_UPLOAD_FILES.value, invalid_input_error_code) + for column in record.columns: + if not isinstance(column.column, str) or not column.column.strip(): + raise SkyflowError(SkyflowMessages.Error.MISSING_COLUMN_NAME_IN_UPLOAD_FILES.value, invalid_input_error_code) + sources = [column.file_path, column.base64, column.file_object] + provided = [source for source in sources if source is not None] + if not provided: + raise SkyflowError(SkyflowMessages.Error.MISSING_FILE_SOURCE_IN_UPLOAD_FILES.value, invalid_input_error_code) + if len(provided) > 1: + raise SkyflowError(SkyflowMessages.Error.MULTIPLE_FILE_SOURCES_IN_UPLOAD_FILES.value, invalid_input_error_code) + if column.base64 is not None and (not isinstance(column.file_name, str) or not column.file_name.strip()): + raise SkyflowError(SkyflowMessages.Error.MISSING_FILE_NAME_FOR_BASE64_IN_UPLOAD_FILES.value, invalid_input_error_code) + + +def validate_delete_files_request(logger, request): + if (not isinstance(request.records, list) or not request.records + or not all(isinstance(r, DeleteFilesRequestRecord) for r in request.records)): + raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_DELETE_FILES.value, invalid_input_error_code) + + for record in request.records: + if not isinstance(record.table_name, str) or not record.table_name.strip(): + raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_DELETE_FILES.value, invalid_input_error_code) + if (not isinstance(record.columns, list) or not record.columns + or not all(isinstance(c, str) and c.strip() for c in record.columns)): + raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMNS_IN_DELETE_FILES.value, invalid_input_error_code) + has_skyflow_id = isinstance(record.skyflow_id, str) and bool(record.skyflow_id.strip()) + has_unique_values = isinstance(record.unique_values, list) and bool(record.unique_values) + if has_skyflow_id == has_unique_values: + raise SkyflowError(SkyflowMessages.Error.INVALID_ID_OR_UNIQUE_VALUES_IN_DELETE_FILES.value, invalid_input_error_code) diff --git a/flowvault/skyflow/vault/client/client.py b/flowvault/skyflow/vault/client/client.py index 557e89d7..d7966404 100644 --- a/flowvault/skyflow/vault/client/client.py +++ b/flowvault/skyflow/vault/client/client.py @@ -26,6 +26,8 @@ ) from skyflow.utils._retry import RetryTransport +CONTENT_TYPE_HEADER = "content-type" + class VaultClient(BaseVaultClient): def __init__(self, config): @@ -100,3 +102,15 @@ def get_records_api(self): def get_tokens_api(self): return self._api_client.tokens + + def get_query_api(self): + return self._api_client.query + + def get_files_api(self): + return self._api_client.files + + def put_signed_url(self, signed_url, content, content_type=None): + headers = {} + if content_type: + headers[CONTENT_TYPE_HEADER] = content_type + return self._sync_httpx_client.put(signed_url, content=content, headers=headers) diff --git a/flowvault/skyflow/vault/controller/_vault.py b/flowvault/skyflow/vault/controller/_vault.py index 1c541644..b388d018 100644 --- a/flowvault/skyflow/vault/controller/_vault.py +++ b/flowvault/skyflow/vault/controller/_vault.py @@ -1,4 +1,7 @@ +import base64 import json +import mimetypes +import os from common.errors import SkyflowError from common.utils import SkyflowMessages as CommonMessages @@ -7,7 +10,11 @@ from common.vault.base_vault_controller import BaseVaultController from skyflow.generated.rest import ( ColumnRedactions, + FileDeleteRecord, + FileUploadColumn, + FileUploadRecord, GetRequestData, + GetTokensFromValuesRequestObject, InsertRecordData, TokenGroupRedactions as WireTokenGroupRedactions, UniqueValue, @@ -16,7 +23,7 @@ ) from skyflow.generated.rest.core import ApiError, ParsingError from skyflow.utils import SkyflowMessages, get_metrics -from skyflow.utils.enums import UpsertType +from skyflow.utils.enums import UpsertType, FileUploadStatus from skyflow.utils._response_parsing import parse_tokens, parse_hashed_data, parse_metadata from skyflow.utils.validations import ( validate_insert_request, @@ -24,6 +31,10 @@ validate_update_request, validate_delete_request, validate_detokenize_request, + validate_query_request, + validate_get_tokens_request, + validate_upload_files_request, + validate_delete_files_request, ) from skyflow.vault.data import ( InsertRequest, @@ -41,11 +52,30 @@ DetokenizeRequest, DetokenizeResponse, DetokenizeResponseRecord, + QueryRequest, + QueryResponse, + QueryResponseRecord, + QueryResponseMetadata, + GetTokensRequest, + GetTokensResponse, + GetTokensResponseRecord, + UploadFilesRequest, + UploadFilesResponse, + UploadFilesResponseRecord, + UploadFilesColumnResult, + DeleteFilesRequest, + DeleteFilesResponse, + DeleteFilesResponseRecord, + DeleteFilesColumnResult, InsertOptions, GetOptions, UpdateOptions, DeleteOptions, DetokenizeOptions, + QueryOptions, + GetTokensOptions, + UploadFilesOptions, + DeleteFilesOptions, RequestContext, ) @@ -57,6 +87,15 @@ OPERATION_UPDATE = "UPDATE" OPERATION_DELETE = "DELETE" OPERATION_DETOKENIZE = "DETOKENIZE" +OPERATION_QUERY = "QUERY" +OPERATION_GET_TOKENS = "GET_TOKENS" +OPERATION_UPLOAD_FILES = "UPLOAD_FILES" +OPERATION_DELETE_FILES = "DELETE_FILES" +FILE_PUT_FAILED_MESSAGE = "PUT failed: {}" +STATUS_KEY = "status" +INVALID_INPUT_ERROR_CODE = CommonMessages.ErrorCodes.INVALID_INPUT.value +HTTP_STATUS_OK = 200 +HTTP_STATUS_MULTIPLE_CHOICES = 300 class VaultController(BaseVaultController): @@ -271,6 +310,123 @@ def detokenize(self, request: DetokenizeRequest, options: DetokenizeOptions = No log_info(SkyflowMessages.Info.DETOKENIZE_SUCCESS.value, self._vault_client.get_logger()) return DetokenizeResponse(records=records) + def query(self, request: QueryRequest, options: QueryOptions = None) -> QueryResponse: + log_info(SkyflowMessages.Info.VALIDATE_QUERY_REQUEST.value, self._vault_client.get_logger()) + validate_query_request(self._vault_client.get_logger(), request) + log_info(SkyflowMessages.Info.QUERY_REQUEST_RESOLVED.value, self._vault_client.get_logger()) + self._vault_client.initialize_client_configuration() + + query_api = self._vault_client.get_query_api() + + try: + log_info(SkyflowMessages.Info.QUERY_TRIGGERED.value, self._vault_client.get_logger()) + raw_response = query_api.with_raw_response.execute_query( + vault_id=self._vault_client.get_vault_id(), + query=request.query, + request_options=self.__unary_request_options(OPERATION_QUERY, options), + ) + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.QUERY_RECORDS_REJECTED.value, self._vault_client.get_logger()) + raise self.__to_skyflow_error(e) + + request_id = self.__extract_request_id(raw_response.headers) + records = [ + QueryResponseRecord(data=self.__wire_record_value(record, 'data', 'data')) + for record in (raw_response.data.records or []) + ] + metadata = self.__to_query_metadata(raw_response.data.metadata) + log_info(SkyflowMessages.Info.QUERY_SUCCESS.value, self._vault_client.get_logger()) + return QueryResponse(records=records, metadata=metadata, request_id=request_id) + + def get_tokens(self, request: GetTokensRequest, options: GetTokensOptions = None) -> GetTokensResponse: + log_info(SkyflowMessages.Info.VALIDATE_GET_TOKENS_REQUEST.value, self._vault_client.get_logger()) + validate_get_tokens_request(self._vault_client.get_logger(), request) + log_info(SkyflowMessages.Info.GET_TOKENS_REQUEST_RESOLVED.value, self._vault_client.get_logger()) + self._vault_client.initialize_client_configuration() + + tokens_api = self._vault_client.get_tokens_api() + + try: + log_info(SkyflowMessages.Info.GET_TOKENS_TRIGGERED.value, self._vault_client.get_logger()) + raw_response = tokens_api.with_raw_response.get_tokens( + vault_id=self._vault_client.get_vault_id(), + records=self.__to_get_tokens_request_data(request.records), + request_options=self.__unary_request_options(OPERATION_GET_TOKENS, options), + ) + request_id = self.__extract_request_id(raw_response.headers) + records = [self.__get_tokens_row(record, request_id) for record in (raw_response.data.records or [])] + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.GET_TOKENS_RECORDS_REJECTED.value, self._vault_client.get_logger()) + error_records, request_id = self.__unary_error_records(e) + if error_records is None: + raise self.__to_skyflow_error(e) + records = [self.__get_tokens_row(record, request_id) for record in error_records] + + log_info(SkyflowMessages.Info.GET_TOKENS_SUCCESS.value, self._vault_client.get_logger()) + return GetTokensResponse(records=records) + + def upload_files(self, request: UploadFilesRequest, options: UploadFilesOptions = None) -> UploadFilesResponse: + log_info(SkyflowMessages.Info.VALIDATE_UPLOAD_FILES_REQUEST.value, self._vault_client.get_logger()) + validate_upload_files_request(self._vault_client.get_logger(), request) + for record in request.records: + self._validate_table_name_if_present(record.table_name) + resolved_records = [self.__resolve_upload_record(record) for record in request.records] + log_info(SkyflowMessages.Info.UPLOAD_FILES_REQUEST_RESOLVED.value, self._vault_client.get_logger()) + self._vault_client.initialize_client_configuration() + + files_api = self._vault_client.get_files_api() + + try: + log_info(SkyflowMessages.Info.UPLOAD_FILES_TRIGGERED.value, self._vault_client.get_logger()) + raw_response = files_api.with_raw_response.upload_files( + vault_id=self._vault_client.get_vault_id(), + records=self.__to_upload_files_request_data(request.records, resolved_records), + request_options=self.__unary_request_options(OPERATION_UPLOAD_FILES, options), + ) + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.UPLOAD_FILES_RECORDS_REJECTED.value, self._vault_client.get_logger()) + raise self.__to_skyflow_error(e) + + request_id = self.__extract_request_id(raw_response.headers) + response_records = raw_response.data.records or [] + records = [] + for index, response_record in enumerate(response_records): + request_record = request.records[index] if index < len(request.records) else None + resolved = resolved_records[index] if index < len(resolved_records) else {} + records.append(self.__upload_files_row(request_record, resolved, response_record, request_id)) + + log_info(SkyflowMessages.Info.UPLOAD_FILES_SUCCESS.value, self._vault_client.get_logger()) + return UploadFilesResponse(records=records) + + def delete_files(self, request: DeleteFilesRequest, options: DeleteFilesOptions = None) -> DeleteFilesResponse: + log_info(SkyflowMessages.Info.VALIDATE_DELETE_FILES_REQUEST.value, self._vault_client.get_logger()) + validate_delete_files_request(self._vault_client.get_logger(), request) + for record in request.records: + self._validate_table_name_if_present(record.table_name) + log_info(SkyflowMessages.Info.DELETE_FILES_REQUEST_RESOLVED.value, self._vault_client.get_logger()) + self._vault_client.initialize_client_configuration() + + files_api = self._vault_client.get_files_api() + + try: + log_info(SkyflowMessages.Info.DELETE_FILES_TRIGGERED.value, self._vault_client.get_logger()) + raw_response = files_api.with_raw_response.delete_files( + vault_id=self._vault_client.get_vault_id(), + records=self.__to_delete_files_request_data(request.records), + request_options=self.__unary_request_options(OPERATION_DELETE_FILES, options), + ) + request_id = self.__extract_request_id(raw_response.headers) + records = [self.__delete_files_row(record, request_id) for record in (raw_response.data.records or [])] + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.DELETE_FILES_RECORDS_REJECTED.value, self._vault_client.get_logger()) + error_records, request_id = self.__unary_error_records(e) + if error_records is None: + raise self.__to_skyflow_error(e) + records = [self.__delete_files_row(record, request_id) for record in error_records] + + log_info(SkyflowMessages.Info.DELETE_FILES_SUCCESS.value, self._vault_client.get_logger()) + return DeleteFilesResponse(records=records) + def __error_body_records(self, e): body = getattr(e, 'body', None) if isinstance(body, dict): @@ -471,5 +627,171 @@ def __detokenize_row(self, resp, request_id=None): request_id=request_id if error is not None else None, ) + def __to_query_metadata(self, metadata): + if metadata is None: + return None + return QueryResponseMetadata(columns=self.__wire_record_value(metadata, 'columns', 'columns')) + + def __to_get_tokens_request_data(self, records): + return [ + GetTokensFromValuesRequestObject(value=record.value, token_group_name=record.token_group_name) + for record in records + ] + + def __get_tokens_row(self, resp, request_id=None): + error = self.__wire_record_value(resp, 'error', 'error') + return GetTokensResponseRecord( + value=self.__wire_record_value(resp, 'value', 'value'), + token_group_name=self.__wire_record_value(resp, 'tokenGroupName', 'token_group_name'), + token=self.__wire_record_value(resp, 'token', 'token'), + http_code=self.__wire_record_value(resp, 'httpCode', 'http_code'), + error=error, + request_id=request_id if error is not None else None, + ) + + def __resolve_upload_record(self, record): + return {column.column: self.__resolve_upload_file(column) for column in record.columns} + + def __resolve_upload_file(self, column): + file_name = column.file_name + if column.file_path is not None: + try: + with open(column.file_path, 'rb') as handle: + content = handle.read() + except OSError: + raise SkyflowError( + SkyflowMessages.Error.FILE_NOT_FOUND_IN_UPLOAD_FILES.value.format(column.file_path), + INVALID_INPUT_ERROR_CODE, + ) + if not file_name: + file_name = os.path.basename(column.file_path) + elif column.base64 is not None: + try: + content = base64.b64decode(column.base64, validate=True) + except (ValueError, TypeError): + raise SkyflowError( + SkyflowMessages.Error.INVALID_BASE64_IN_UPLOAD_FILES.value.format(column.column), + INVALID_INPUT_ERROR_CODE, + ) + else: + content = column.file_object.read() + if isinstance(content, str): + content = content.encode('utf-8') + if not file_name: + object_name = getattr(column.file_object, 'name', None) + if object_name: + file_name = os.path.basename(object_name) + content_type = column.content_type + if content_type is None and file_name: + content_type = mimetypes.guess_type(file_name)[0] + return file_name, content_type, content + + def __to_upload_files_request_data(self, records, resolved_records): + wire_records = [] + for index, record in enumerate(records): + resolved = resolved_records[index] + columns = [ + FileUploadColumn(column=column.column, **self.__omit_none(file_name=resolved[column.column][0])) + for column in record.columns + ] + wire_records.append(FileUploadRecord( + table_name=record.table_name, + columns=columns, + **self.__omit_none(skyflow_id=record.skyflow_id), + )) + return wire_records + + def __upload_files_row(self, request_record, resolved, response_record, request_id): + error = self.__wire_record_value(response_record, 'error', 'error') + signed_urls = self.__wire_record_value(response_record, 'data', 'data') + columns = [] + for column in (request_record.columns if request_record is not None else []): + resolved_file = resolved.get(column.column) + if error is not None: + columns.append(UploadFilesColumnResult( + column=column.column, + file_name=resolved_file[0] if resolved_file else None, + upload_status=FileUploadStatus.SKIPPED.value, + error=error, + )) + continue + signed_url = signed_urls.get(column.column) if isinstance(signed_urls, dict) else None + columns.append(self.__upload_column(column.column, resolved_file, signed_url)) + return UploadFilesResponseRecord( + skyflow_id=self.__wire_record_value(response_record, 'skyflowID', 'skyflow_id'), + table_name=self.__wire_record_value(response_record, 'tableName', 'table_name'), + columns=columns, + http_code=self.__wire_record_value(response_record, 'httpCode', 'http_code'), + error=error, + request_id=request_id if error is not None else None, + ) + + def __upload_column(self, column_name, resolved_file, signed_url): + file_name, content_type, content = resolved_file if resolved_file else (None, None, None) + if not signed_url: + return UploadFilesColumnResult( + column=column_name, + file_name=file_name, + upload_status=FileUploadStatus.SKIPPED.value, + ) + try: + put_response = self._vault_client.put_signed_url(signed_url, content, content_type) + status_code = put_response.status_code + except Exception as e: + return UploadFilesColumnResult( + column=column_name, + file_name=file_name, + upload_status=FileUploadStatus.FAILED.value, + error=str(e), + ) + if HTTP_STATUS_OK <= status_code < HTTP_STATUS_MULTIPLE_CHOICES: + return UploadFilesColumnResult( + column=column_name, + file_name=file_name, + upload_status=FileUploadStatus.UPLOADED.value, + ) + return UploadFilesColumnResult( + column=column_name, + file_name=file_name, + upload_status=FileUploadStatus.FAILED.value, + error=FILE_PUT_FAILED_MESSAGE.format(status_code), + ) + + def __to_delete_files_request_data(self, records): + return [ + FileDeleteRecord( + table_name=record.table_name, + columns=record.columns, + **self.__omit_none( + skyflow_id=record.skyflow_id, + unique_values=self.__to_unique_values(record.unique_values), + ), + ) + for record in records + ] + + def __delete_files_row(self, resp, request_id=None): + error = self.__wire_record_value(resp, 'error', 'error') + data = self.__wire_record_value(resp, 'data', 'data') + columns = None + if isinstance(data, dict): + columns = [ + DeleteFilesColumnResult(column=name, status=self.__file_column_status(value)) + for name, value in data.items() + ] + return DeleteFilesResponseRecord( + skyflow_id=self.__wire_record_value(resp, 'skyflowID', 'skyflow_id'), + table_name=self.__wire_record_value(resp, 'tableName', 'table_name'), + columns=columns, + http_code=self.__wire_record_value(resp, 'httpCode', 'http_code'), + error=error, + request_id=request_id if error is not None else None, + ) + + def __file_column_status(self, value): + if isinstance(value, dict): + return value.get(STATUS_KEY) + return value + diff --git a/flowvault/skyflow/vault/data/__init__.py b/flowvault/skyflow/vault/data/__init__.py index 11e00a73..511f084e 100644 --- a/flowvault/skyflow/vault/data/__init__.py +++ b/flowvault/skyflow/vault/data/__init__.py @@ -27,4 +27,27 @@ from ._update_options import UpdateOptions from ._delete_options import DeleteOptions from ._detokenize_options import DetokenizeOptions +from ._query_request import QueryRequest +from ._query_options import QueryOptions +from ._query_response import QueryResponse +from ._query_response_record import QueryResponseRecord +from ._query_response_metadata import QueryResponseMetadata +from ._get_tokens_request_record import GetTokensRequestRecord +from ._get_tokens_request import GetTokensRequest +from ._get_tokens_options import GetTokensOptions +from ._get_tokens_response import GetTokensResponse +from ._get_tokens_response_record import GetTokensResponseRecord +from ._upload_files_request_column import UploadFilesRequestColumn +from ._upload_files_request_record import UploadFilesRequestRecord +from ._upload_files_request import UploadFilesRequest +from ._upload_files_options import UploadFilesOptions +from ._upload_files_column_result import UploadFilesColumnResult +from ._upload_files_response_record import UploadFilesResponseRecord +from ._upload_files_response import UploadFilesResponse +from ._delete_files_request_record import DeleteFilesRequestRecord +from ._delete_files_request import DeleteFilesRequest +from ._delete_files_options import DeleteFilesOptions +from ._delete_files_column_result import DeleteFilesColumnResult +from ._delete_files_response_record import DeleteFilesResponseRecord +from ._delete_files_response import DeleteFilesResponse from skyflow.utils.enums import CustomHeaderKey diff --git a/flowvault/skyflow/vault/data/_delete_files_column_result.py b/flowvault/skyflow/vault/data/_delete_files_column_result.py new file mode 100644 index 00000000..0f67785d --- /dev/null +++ b/flowvault/skyflow/vault/data/_delete_files_column_result.py @@ -0,0 +1,10 @@ +class DeleteFilesColumnResult: + def __init__(self, column: str, status: str = None): + self.column = column + self.status = status + + def __repr__(self): + return f"DeleteFilesColumnResult(column={self.column}, status={self.status})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow/vault/data/_delete_files_options.py b/flowvault/skyflow/vault/data/_delete_files_options.py new file mode 100644 index 00000000..b7641f6b --- /dev/null +++ b/flowvault/skyflow/vault/data/_delete_files_options.py @@ -0,0 +1,8 @@ +from typing import Callable, Optional + +from ._request_context import RequestContext + + +class DeleteFilesOptions: + def __init__(self, interceptor: Optional[Callable[[RequestContext], None]] = None): + self.interceptor = interceptor diff --git a/flowvault/skyflow/vault/data/_delete_files_request.py b/flowvault/skyflow/vault/data/_delete_files_request.py new file mode 100644 index 00000000..95dc4683 --- /dev/null +++ b/flowvault/skyflow/vault/data/_delete_files_request.py @@ -0,0 +1,9 @@ +from typing import List + +from common.vault.data import BaseDeleteFilesRequest +from ._delete_files_request_record import DeleteFilesRequestRecord + + +class DeleteFilesRequest(BaseDeleteFilesRequest): + def __init__(self, records: List[DeleteFilesRequestRecord]): + super().__init__(records) diff --git a/flowvault/skyflow/vault/data/_delete_files_request_record.py b/flowvault/skyflow/vault/data/_delete_files_request_record.py new file mode 100644 index 00000000..aaa25c4e --- /dev/null +++ b/flowvault/skyflow/vault/data/_delete_files_request_record.py @@ -0,0 +1,9 @@ +from typing import List + + +class DeleteFilesRequestRecord: + def __init__(self, table_name: str, columns: List[str], skyflow_id: str = None, unique_values: List[dict] = None): + self.table_name = table_name + self.columns = columns + self.skyflow_id = skyflow_id + self.unique_values = unique_values diff --git a/flowvault/skyflow/vault/data/_delete_files_response.py b/flowvault/skyflow/vault/data/_delete_files_response.py new file mode 100644 index 00000000..c77c8861 --- /dev/null +++ b/flowvault/skyflow/vault/data/_delete_files_response.py @@ -0,0 +1,6 @@ +from common.vault.data import BaseDeleteFilesResponse + + +class DeleteFilesResponse(BaseDeleteFilesResponse): + def __init__(self, records=None): + super().__init__(records) diff --git a/flowvault/skyflow/vault/data/_delete_files_response_record.py b/flowvault/skyflow/vault/data/_delete_files_response_record.py new file mode 100644 index 00000000..a09624f1 --- /dev/null +++ b/flowvault/skyflow/vault/data/_delete_files_response_record.py @@ -0,0 +1,15 @@ +class DeleteFilesResponseRecord: + def __init__(self, skyflow_id: str = None, table_name: str = None, columns=None, + http_code: int = None, error: str = None, request_id: str = None): + self.skyflow_id = skyflow_id + self.table_name = table_name + self.columns = columns + self.http_code = http_code + self.error = error + self.request_id = request_id + + def __repr__(self): + return ("DeleteFilesResponseRecord(skyflow_id={}, table_name={}, columns={}, " + "http_code={}, error={}, request_id={})").format( + self.skyflow_id, self.table_name, self.columns, + self.http_code, self.error, self.request_id) diff --git a/flowvault/skyflow/vault/data/_get_tokens_options.py b/flowvault/skyflow/vault/data/_get_tokens_options.py new file mode 100644 index 00000000..242d4bc0 --- /dev/null +++ b/flowvault/skyflow/vault/data/_get_tokens_options.py @@ -0,0 +1,8 @@ +from typing import Callable, Optional + +from ._request_context import RequestContext + + +class GetTokensOptions: + def __init__(self, interceptor: Optional[Callable[[RequestContext], None]] = None): + self.interceptor = interceptor diff --git a/flowvault/skyflow/vault/data/_get_tokens_request.py b/flowvault/skyflow/vault/data/_get_tokens_request.py new file mode 100644 index 00000000..444cb56e --- /dev/null +++ b/flowvault/skyflow/vault/data/_get_tokens_request.py @@ -0,0 +1,9 @@ +from typing import List + +from common.vault.data import BaseGetTokensRequest +from ._get_tokens_request_record import GetTokensRequestRecord + + +class GetTokensRequest(BaseGetTokensRequest): + def __init__(self, records: List[GetTokensRequestRecord]): + super().__init__(records) diff --git a/flowvault/skyflow/vault/data/_get_tokens_request_record.py b/flowvault/skyflow/vault/data/_get_tokens_request_record.py new file mode 100644 index 00000000..ffb11f58 --- /dev/null +++ b/flowvault/skyflow/vault/data/_get_tokens_request_record.py @@ -0,0 +1,4 @@ +class GetTokensRequestRecord: + def __init__(self, value, token_group_name: str): + self.value = value + self.token_group_name = token_group_name diff --git a/flowvault/skyflow/vault/data/_get_tokens_response.py b/flowvault/skyflow/vault/data/_get_tokens_response.py new file mode 100644 index 00000000..c88da447 --- /dev/null +++ b/flowvault/skyflow/vault/data/_get_tokens_response.py @@ -0,0 +1,6 @@ +from common.vault.data import BaseGetTokensResponse + + +class GetTokensResponse(BaseGetTokensResponse): + def __init__(self, records=None): + super().__init__(records) diff --git a/flowvault/skyflow/vault/data/_get_tokens_response_record.py b/flowvault/skyflow/vault/data/_get_tokens_response_record.py new file mode 100644 index 00000000..4a6425c6 --- /dev/null +++ b/flowvault/skyflow/vault/data/_get_tokens_response_record.py @@ -0,0 +1,15 @@ +class GetTokensResponseRecord: + def __init__(self, value=None, token_group_name=None, token=None, + http_code=None, error=None, request_id=None): + self.value = value + self.token_group_name = token_group_name + self.token = token + self.http_code = http_code + self.error = error + self.request_id = request_id + + def __repr__(self): + return ("GetTokensResponseRecord(value={}, token_group_name={}, token={}, " + "http_code={}, error={}, request_id={})").format( + self.value, self.token_group_name, self.token, + self.http_code, self.error, self.request_id) diff --git a/flowvault/skyflow/vault/data/_query_options.py b/flowvault/skyflow/vault/data/_query_options.py new file mode 100644 index 00000000..febda3a2 --- /dev/null +++ b/flowvault/skyflow/vault/data/_query_options.py @@ -0,0 +1,8 @@ +from typing import Callable, Optional + +from ._request_context import RequestContext + + +class QueryOptions: + def __init__(self, interceptor: Optional[Callable[[RequestContext], None]] = None): + self.interceptor = interceptor diff --git a/flowvault/skyflow/vault/data/_query_request.py b/flowvault/skyflow/vault/data/_query_request.py new file mode 100644 index 00000000..bec0658f --- /dev/null +++ b/flowvault/skyflow/vault/data/_query_request.py @@ -0,0 +1,6 @@ +from common.vault.data import BaseQueryRequest + + +class QueryRequest(BaseQueryRequest): + def __init__(self, query: str): + super().__init__(query) diff --git a/flowvault/skyflow/vault/data/_query_response.py b/flowvault/skyflow/vault/data/_query_response.py new file mode 100644 index 00000000..bf92a30a --- /dev/null +++ b/flowvault/skyflow/vault/data/_query_response.py @@ -0,0 +1,14 @@ +from common.vault.data import BaseQueryResponse + + +class QueryResponse(BaseQueryResponse): + def __init__(self, records=None, metadata=None, request_id=None): + super().__init__(records) + self.metadata = metadata + self.request_id = request_id + + def __repr__(self): + return f"QueryResponse(records={self.records}, metadata={self.metadata}, request_id={self.request_id})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow/vault/data/_query_response_metadata.py b/flowvault/skyflow/vault/data/_query_response_metadata.py new file mode 100644 index 00000000..37fc0ed4 --- /dev/null +++ b/flowvault/skyflow/vault/data/_query_response_metadata.py @@ -0,0 +1,9 @@ +class QueryResponseMetadata: + def __init__(self, columns: list = None): + self.columns = columns + + def __repr__(self): + return f"QueryResponseMetadata(columns={self.columns})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow/vault/data/_query_response_record.py b/flowvault/skyflow/vault/data/_query_response_record.py new file mode 100644 index 00000000..6281a2c6 --- /dev/null +++ b/flowvault/skyflow/vault/data/_query_response_record.py @@ -0,0 +1,9 @@ +class QueryResponseRecord: + def __init__(self, data: dict = None): + self.data = data + + def __repr__(self): + return f"QueryResponseRecord(data={self.data})" + + def __str__(self): + return self.__repr__() diff --git a/flowvault/skyflow/vault/data/_upload_files_column_result.py b/flowvault/skyflow/vault/data/_upload_files_column_result.py new file mode 100644 index 00000000..33b9d219 --- /dev/null +++ b/flowvault/skyflow/vault/data/_upload_files_column_result.py @@ -0,0 +1,11 @@ +class UploadFilesColumnResult: + def __init__(self, column: str, file_name: str = None, upload_status: str = None, error: str = None): + self.column = column + self.file_name = file_name + self.upload_status = upload_status + self.error = error + + def __repr__(self): + return ("UploadFilesColumnResult(column={}, file_name={}, upload_status={}, " + "error={})").format( + self.column, self.file_name, self.upload_status, self.error) diff --git a/flowvault/skyflow/vault/data/_upload_files_options.py b/flowvault/skyflow/vault/data/_upload_files_options.py new file mode 100644 index 00000000..7b73c502 --- /dev/null +++ b/flowvault/skyflow/vault/data/_upload_files_options.py @@ -0,0 +1,8 @@ +from typing import Callable, Optional + +from ._request_context import RequestContext + + +class UploadFilesOptions: + def __init__(self, interceptor: Optional[Callable[[RequestContext], None]] = None): + self.interceptor = interceptor diff --git a/flowvault/skyflow/vault/data/_upload_files_request.py b/flowvault/skyflow/vault/data/_upload_files_request.py new file mode 100644 index 00000000..a68a43db --- /dev/null +++ b/flowvault/skyflow/vault/data/_upload_files_request.py @@ -0,0 +1,9 @@ +from typing import List + +from common.vault.data import BaseUploadFilesRequest +from ._upload_files_request_record import UploadFilesRequestRecord + + +class UploadFilesRequest(BaseUploadFilesRequest): + def __init__(self, records: List[UploadFilesRequestRecord]): + super().__init__(records) diff --git a/flowvault/skyflow/vault/data/_upload_files_request_column.py b/flowvault/skyflow/vault/data/_upload_files_request_column.py new file mode 100644 index 00000000..0380fd26 --- /dev/null +++ b/flowvault/skyflow/vault/data/_upload_files_request_column.py @@ -0,0 +1,12 @@ +from typing import BinaryIO + + +class UploadFilesRequestColumn: + def __init__(self, column: str, file_path: str = None, base64: str = None, + file_object: BinaryIO = None, file_name: str = None, content_type: str = None): + self.column = column + self.file_path = file_path + self.base64 = base64 + self.file_object = file_object + self.file_name = file_name + self.content_type = content_type diff --git a/flowvault/skyflow/vault/data/_upload_files_request_record.py b/flowvault/skyflow/vault/data/_upload_files_request_record.py new file mode 100644 index 00000000..0f2a8d61 --- /dev/null +++ b/flowvault/skyflow/vault/data/_upload_files_request_record.py @@ -0,0 +1,10 @@ +from typing import List + +from ._upload_files_request_column import UploadFilesRequestColumn + + +class UploadFilesRequestRecord: + def __init__(self, table_name: str, columns: List[UploadFilesRequestColumn], skyflow_id: str = None): + self.table_name = table_name + self.columns = columns + self.skyflow_id = skyflow_id diff --git a/flowvault/skyflow/vault/data/_upload_files_response.py b/flowvault/skyflow/vault/data/_upload_files_response.py new file mode 100644 index 00000000..53cdb3a9 --- /dev/null +++ b/flowvault/skyflow/vault/data/_upload_files_response.py @@ -0,0 +1,6 @@ +from common.vault.data import BaseUploadFilesResponse + + +class UploadFilesResponse(BaseUploadFilesResponse): + def __init__(self, records=None): + super().__init__(records) diff --git a/flowvault/skyflow/vault/data/_upload_files_response_record.py b/flowvault/skyflow/vault/data/_upload_files_response_record.py new file mode 100644 index 00000000..9fcb9c9c --- /dev/null +++ b/flowvault/skyflow/vault/data/_upload_files_response_record.py @@ -0,0 +1,15 @@ +class UploadFilesResponseRecord: + def __init__(self, skyflow_id: str = None, table_name: str = None, columns=None, + http_code: int = None, error: str = None, request_id: str = None): + self.skyflow_id = skyflow_id + self.table_name = table_name + self.columns = columns + self.http_code = http_code + self.error = error + self.request_id = request_id + + def __repr__(self): + return ("UploadFilesResponseRecord(skyflow_id={}, table_name={}, columns={}, " + "http_code={}, error={}, request_id={})").format( + self.skyflow_id, self.table_name, self.columns, + self.http_code, self.error, self.request_id) diff --git a/flowvault/tests/utils/validations/test__validations_new_ops.py b/flowvault/tests/utils/validations/test__validations_new_ops.py new file mode 100644 index 00000000..6c2e4e36 --- /dev/null +++ b/flowvault/tests/utils/validations/test__validations_new_ops.py @@ -0,0 +1,178 @@ +import unittest + +from common.errors import SkyflowError +from skyflow.utils.validations import ( + validate_query_request, + validate_get_tokens_request, + validate_upload_files_request, + validate_delete_files_request, +) +from skyflow.vault.data import ( + QueryRequest, + GetTokensRequest, + GetTokensRequestRecord, + UploadFilesRequest, + UploadFilesRequestRecord, + UploadFilesRequestColumn, + DeleteFilesRequest, + DeleteFilesRequestRecord, +) + + +class TestValidateQueryRequest(unittest.TestCase): + def test_valid(self): + validate_query_request(None, QueryRequest(query="SELECT 1")) + + def test_blank_raises(self): + with self.assertRaises(SkyflowError): + validate_query_request(None, QueryRequest(query=" ")) + + def test_empty_raises(self): + with self.assertRaises(SkyflowError): + validate_query_request(None, QueryRequest(query="")) + + def test_non_string_raises(self): + with self.assertRaises(SkyflowError): + validate_query_request(None, QueryRequest(query=123)) + + +class TestValidateGetTokensRequest(unittest.TestCase): + def test_valid(self): + validate_get_tokens_request(None, GetTokensRequest(records=[ + GetTokensRequestRecord(value="v", token_group_name="g"), + ])) + + def test_empty_records_raises(self): + with self.assertRaises(SkyflowError): + validate_get_tokens_request(None, GetTokensRequest(records=[])) + + def test_non_record_item_raises(self): + with self.assertRaises(SkyflowError): + validate_get_tokens_request(None, GetTokensRequest(records=["not-a-record"])) + + def test_none_value_raises(self): + with self.assertRaises(SkyflowError): + validate_get_tokens_request(None, GetTokensRequest(records=[ + GetTokensRequestRecord(value=None, token_group_name="g"), + ])) + + def test_blank_token_group_name_raises(self): + with self.assertRaises(SkyflowError): + validate_get_tokens_request(None, GetTokensRequest(records=[ + GetTokensRequestRecord(value="v", token_group_name=" "), + ])) + + +class TestValidateUploadFilesRequest(unittest.TestCase): + def _column(self, **kwargs): + kwargs.setdefault("column", "c") + kwargs.setdefault("file_path", "/tmp/x") + return UploadFilesRequestColumn(**kwargs) + + def test_valid(self): + validate_upload_files_request(None, UploadFilesRequest(records=[ + UploadFilesRequestRecord(table_name="t1", columns=[self._column()]), + ])) + + def test_empty_records_raises(self): + with self.assertRaises(SkyflowError): + validate_upload_files_request(None, UploadFilesRequest(records=[])) + + def test_missing_table_name_raises(self): + with self.assertRaises(SkyflowError): + validate_upload_files_request(None, UploadFilesRequest(records=[ + UploadFilesRequestRecord(table_name=" ", columns=[self._column()]), + ])) + + def test_empty_columns_raises(self): + with self.assertRaises(SkyflowError): + validate_upload_files_request(None, UploadFilesRequest(records=[ + UploadFilesRequestRecord(table_name="t1", columns=[]), + ])) + + def test_missing_column_name_raises(self): + with self.assertRaises(SkyflowError): + validate_upload_files_request(None, UploadFilesRequest(records=[ + UploadFilesRequestRecord(table_name="t1", columns=[ + UploadFilesRequestColumn(column=" ", file_path="/tmp/x"), + ]), + ])) + + def test_no_file_source_raises(self): + with self.assertRaises(SkyflowError): + validate_upload_files_request(None, UploadFilesRequest(records=[ + UploadFilesRequestRecord(table_name="t1", columns=[UploadFilesRequestColumn(column="c")]), + ])) + + def test_multiple_file_sources_raises(self): + with self.assertRaises(SkyflowError): + validate_upload_files_request(None, UploadFilesRequest(records=[ + UploadFilesRequestRecord(table_name="t1", columns=[ + UploadFilesRequestColumn(column="c", file_path="/tmp/x", base64="abc"), + ]), + ])) + + def test_base64_without_file_name_raises(self): + with self.assertRaises(SkyflowError): + validate_upload_files_request(None, UploadFilesRequest(records=[ + UploadFilesRequestRecord(table_name="t1", columns=[ + UploadFilesRequestColumn(column="c", base64="abc"), + ]), + ])) + + def test_base64_with_file_name_valid(self): + validate_upload_files_request(None, UploadFilesRequest(records=[ + UploadFilesRequestRecord(table_name="t1", columns=[ + UploadFilesRequestColumn(column="c", base64="abc", file_name="x.bin"), + ]), + ])) + + +class TestValidateDeleteFilesRequest(unittest.TestCase): + def test_valid_by_skyflow_id(self): + validate_delete_files_request(None, DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c"], skyflow_id="id1"), + ])) + + def test_valid_by_unique_values(self): + validate_delete_files_request(None, DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c"], unique_values=[{"email": "a@b.com"}]), + ])) + + def test_empty_records_raises(self): + with self.assertRaises(SkyflowError): + validate_delete_files_request(None, DeleteFilesRequest(records=[])) + + def test_missing_table_name_raises(self): + with self.assertRaises(SkyflowError): + validate_delete_files_request(None, DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name=" ", columns=["c"], skyflow_id="id1"), + ])) + + def test_empty_columns_raises(self): + with self.assertRaises(SkyflowError): + validate_delete_files_request(None, DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=[], skyflow_id="id1"), + ])) + + def test_non_string_columns_raises(self): + with self.assertRaises(SkyflowError): + validate_delete_files_request(None, DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=[1, 2], skyflow_id="id1"), + ])) + + def test_neither_selector_raises(self): + with self.assertRaises(SkyflowError): + validate_delete_files_request(None, DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c"]), + ])) + + def test_both_selectors_raises(self): + with self.assertRaises(SkyflowError): + validate_delete_files_request(None, DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c"], skyflow_id="id1", unique_values=[{"email": "a@b.com"}]), + ])) + + +if __name__ == "__main__": + unittest.main() diff --git a/flowvault/tests/vault/client/test__client_file_query_apis.py b/flowvault/tests/vault/client/test__client_file_query_apis.py new file mode 100644 index 00000000..af048691 --- /dev/null +++ b/flowvault/tests/vault/client/test__client_file_query_apis.py @@ -0,0 +1,44 @@ +import unittest +from unittest.mock import MagicMock + +from skyflow.vault.client.client import VaultClient + + +class TestVaultClientFileAndQueryApis(unittest.TestCase): + def setUp(self): + self.vault_client = VaultClient({"vault_id": "test_vault"}) + + def test_get_query_api_returns_query(self): + self.vault_client._api_client = MagicMock() + self.assertEqual(self.vault_client.get_query_api(), self.vault_client._api_client.query) + + def test_get_files_api_returns_files(self): + self.vault_client._api_client = MagicMock() + self.assertEqual(self.vault_client.get_files_api(), self.vault_client._api_client.files) + + def test_put_signed_url_sends_content_and_content_type(self): + http_client = MagicMock() + http_client.put.return_value = "put-result" + self.vault_client._sync_httpx_client = http_client + + result = self.vault_client.put_signed_url("https://signed/url", b"bytes", "application/pdf") + + self.assertEqual(result, "put-result") + _, kwargs = http_client.put.call_args + args, _ = http_client.put.call_args + self.assertEqual(args[0], "https://signed/url") + self.assertEqual(kwargs["content"], b"bytes") + self.assertEqual(kwargs["headers"], {"content-type": "application/pdf"}) + + def test_put_signed_url_without_content_type_sends_empty_headers(self): + http_client = MagicMock() + self.vault_client._sync_httpx_client = http_client + + self.vault_client.put_signed_url("https://signed/url", b"bytes") + + _, kwargs = http_client.put.call_args + self.assertEqual(kwargs["headers"], {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/flowvault/tests/vault/controller/test__vault_files_query_tokens.py b/flowvault/tests/vault/controller/test__vault_files_query_tokens.py new file mode 100644 index 00000000..f856be65 --- /dev/null +++ b/flowvault/tests/vault/controller/test__vault_files_query_tokens.py @@ -0,0 +1,561 @@ +import base64 +import os +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock, patch + +from common.errors import SkyflowError +from skyflow.generated.rest.core import ApiError +from skyflow.utils.enums import CustomHeaderKey, FileUploadStatus +from skyflow.vault.controller import VaultController +from skyflow.vault.data import ( + QueryRequest, + QueryOptions, + GetTokensRequest, + GetTokensRequestRecord, + UploadFilesRequest, + UploadFilesRequestRecord, + UploadFilesRequestColumn, + UploadFilesOptions, + DeleteFilesRequest, + DeleteFilesRequestRecord, +) + + +def fake_query_raw_response(records, metadata, headers=None): + return SimpleNamespace(data=SimpleNamespace(records=records, metadata=metadata), headers=headers or {}) + + +class FakeQueryRecord: + def __init__(self, data): + self.data = data + + +class FakeQueryMetadata: + def __init__(self, columns): + self.columns = columns + + +class FakeTokenizeResponseObject: + def __init__(self, value=None, token_group_name=None, token=None, http_code=None, error=None): + self.value = value + self.token_group_name = token_group_name + self.token = token + self.http_code = http_code + self.error = error + + +def fake_records_raw_response(records, headers=None): + return SimpleNamespace(data=SimpleNamespace(records=records), headers=headers or {}) + + +class FakeFileUploadResponseObject: + def __init__(self, skyflow_id=None, table_name=None, data=None, error=None, http_code=None): + self.skyflow_id = skyflow_id + self.table_name = table_name + self.data = data + self.error = error + self.http_code = http_code + + +class FakeFileDeleteResponseObject: + def __init__(self, skyflow_id=None, table_name=None, data=None, error=None, http_code=None): + self.skyflow_id = skyflow_id + self.table_name = table_name + self.data = data + self.error = error + self.http_code = http_code + + +class TestVaultQuery(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.query_api = MagicMock() + self.vault_client.get_query_api.return_value = self.query_api + self.vault = VaultController(self.vault_client) + + @patch("skyflow.vault.controller._vault.validate_query_request") + def test_query_validates_before_initializing_client(self, mock_validate): + self.query_api.with_raw_response.execute_query.return_value = fake_query_raw_response([], None) + request = QueryRequest(query="SELECT a FROM t1") + + self.vault.query(request) + + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.vault_client.initialize_client_configuration.assert_called_once() + + def test_query_raises_for_invalid_request(self): + with self.assertRaises(SkyflowError): + self.vault.query(QueryRequest(query=" ")) + self.vault_client.initialize_client_configuration.assert_not_called() + + def test_maps_query_and_vault_id(self): + self.query_api.with_raw_response.execute_query.return_value = fake_query_raw_response([], None) + + self.vault.query(QueryRequest(query="SELECT a FROM t1")) + + _, kwargs = self.query_api.with_raw_response.execute_query.call_args + self.assertEqual(kwargs["vault_id"], "vault123") + self.assertEqual(kwargs["query"], "SELECT a FROM t1") + + def test_maps_records_metadata_and_request_id(self): + self.query_api.with_raw_response.execute_query.return_value = fake_query_raw_response( + [FakeQueryRecord({"name": "john"}), FakeQueryRecord({"name": "jane"})], + FakeQueryMetadata(["name"]), + headers={"x-request-id": "req-q"}, + ) + + response = self.vault.query(QueryRequest(query="SELECT name FROM t1")) + + self.assertEqual([r.data for r in response.records], [{"name": "john"}, {"name": "jane"}]) + self.assertEqual(response.metadata.columns, ["name"]) + self.assertEqual(response.request_id, "req-q") + + def test_metadata_none_yields_none_metadata(self): + self.query_api.with_raw_response.execute_query.return_value = fake_query_raw_response( + [FakeQueryRecord({"a": 1})], None, + ) + + response = self.vault.query(QueryRequest(query="SELECT a FROM t1")) + + self.assertIsNone(response.metadata) + + def test_transport_exception_raises_skyflow_error(self): + self.query_api.with_raw_response.execute_query.side_effect = Exception("network blip") + with self.assertRaises(SkyflowError) as ctx: + self.vault.query(QueryRequest(query="SELECT a FROM t1")) + self.assertIn("network blip", ctx.exception.message) + + def test_api_error_raises_with_status(self): + self.query_api.with_raw_response.execute_query.side_effect = ApiError( + status_code=400, headers={"x-request-id": "req-e"}, body={"error": "bad sql"}, + ) + with self.assertRaises(SkyflowError) as ctx: + self.vault.query(QueryRequest(query="SELECT a FROM t1")) + self.assertEqual(ctx.exception.http_code, 400) + + def test_interceptor_adds_custom_header(self): + self.query_api.with_raw_response.execute_query.return_value = fake_query_raw_response([], None) + + def interceptor(context): + context.add_header(CustomHeaderKey.REQUEST_ID_HEADER, "req-x") + + self.vault.query(QueryRequest(query="SELECT a FROM t1"), QueryOptions(interceptor=interceptor)) + + _, kwargs = self.query_api.with_raw_response.execute_query.call_args + self.assertEqual(kwargs["request_options"]["additional_headers"]["x-request-id"], "req-x") + + +class TestVaultGetTokens(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.tokens_api = MagicMock() + self.vault_client.get_tokens_api.return_value = self.tokens_api + self.vault = VaultController(self.vault_client) + + @patch("skyflow.vault.controller._vault.validate_get_tokens_request") + def test_validates_before_initializing_client(self, mock_validate): + self.tokens_api.with_raw_response.get_tokens.return_value = fake_records_raw_response([]) + request = GetTokensRequest(records=[GetTokensRequestRecord(value="v", token_group_name="g")]) + + self.vault.get_tokens(request) + + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.vault_client.initialize_client_configuration.assert_called_once() + + def test_raises_for_invalid_request(self): + with self.assertRaises(SkyflowError): + self.vault.get_tokens(GetTokensRequest(records=[])) + self.vault_client.initialize_client_configuration.assert_not_called() + + def test_maps_records(self): + self.tokens_api.with_raw_response.get_tokens.return_value = fake_records_raw_response([]) + + self.vault.get_tokens(GetTokensRequest(records=[ + GetTokensRequestRecord(value="v1", token_group_name="g1"), + GetTokensRequestRecord(value="v2", token_group_name="g2"), + ])) + + _, kwargs = self.tokens_api.with_raw_response.get_tokens.call_args + self.assertEqual(kwargs["vault_id"], "vault123") + self.assertEqual(len(kwargs["records"]), 2) + self.assertEqual(kwargs["records"][0].value, "v1") + self.assertEqual(kwargs["records"][0].token_group_name, "g1") + + def test_success_and_error_records_in_one_list(self): + self.tokens_api.with_raw_response.get_tokens.return_value = fake_records_raw_response([ + FakeTokenizeResponseObject(value="v1", token_group_name="g1", token="tok1", http_code=200), + FakeTokenizeResponseObject(value="v2", token_group_name="g1", token=None, http_code=404, error="Token not found."), + ], headers={"x-request-id": "req-t"}) + + response = self.vault.get_tokens(GetTokensRequest(records=[ + GetTokensRequestRecord(value="v1", token_group_name="g1"), + GetTokensRequestRecord(value="v2", token_group_name="g1"), + ])) + + self.assertEqual(response.records[0].token, "tok1") + self.assertIsNone(response.records[0].error) + self.assertIsNone(response.records[0].request_id) + self.assertEqual(response.records[1].error, "Token not found.") + self.assertEqual(response.records[1].http_code, 404) + self.assertEqual(response.records[1].request_id, "req-t") + + def test_transport_exception_raises_skyflow_error(self): + self.tokens_api.with_raw_response.get_tokens.side_effect = Exception("network blip") + with self.assertRaises(SkyflowError) as ctx: + self.vault.get_tokens(GetTokensRequest(records=[GetTokensRequestRecord(value="v", token_group_name="g")])) + self.assertIn("network blip", ctx.exception.message) + + def test_api_error_with_per_record_body_returns_error_row(self): + self.tokens_api.with_raw_response.get_tokens.side_effect = ApiError( + status_code=404, headers={"x-request-id": "req-3"}, + body={"records": [{"error": "Token not found.", "httpCode": 404}]}, + ) + + response = self.vault.get_tokens(GetTokensRequest(records=[GetTokensRequestRecord(value="v", token_group_name="g")])) + self.assertEqual(response.records[0].error, "Token not found.") + self.assertEqual(response.records[0].http_code, 404) + self.assertEqual(response.records[0].request_id, "req-3") + + def test_injects_authorization_header_from_current_bearer_token(self): + self.vault_client.get_current_bearer_token.return_value = "the-current-token" + self.tokens_api.with_raw_response.get_tokens.return_value = fake_records_raw_response([]) + + self.vault.get_tokens(GetTokensRequest(records=[GetTokensRequestRecord(value="v", token_group_name="g")])) + + _, kwargs = self.tokens_api.with_raw_response.get_tokens.call_args + self.assertEqual(kwargs["request_options"]["additional_headers"].get("Authorization"), "Bearer the-current-token") + + +class TestVaultUploadFiles(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.files_api = MagicMock() + self.vault_client.get_files_api.return_value = self.files_api + self.vault_client.put_signed_url.return_value = SimpleNamespace(status_code=200) + self.vault = VaultController(self.vault_client) + self._tempfiles = [] + + def tearDown(self): + for path in self._tempfiles: + try: + os.unlink(path) + except OSError: + pass + + def _make_file(self, content=b"filedata", suffix=".txt"): + fd, path = tempfile.mkstemp(suffix=suffix) + with os.fdopen(fd, "wb") as handle: + handle.write(content) + self._tempfiles.append(path) + return path + + def _ok_response(self, column="c", url="https://signed/url", skyflow_id="sid1", headers=None): + return fake_records_raw_response( + [FakeFileUploadResponseObject(skyflow_id=skyflow_id, table_name="t1", data={column: url}, http_code=200)], + headers=headers, + ) + + @patch("skyflow.vault.controller._vault.validate_upload_files_request") + def test_validates_before_initializing_client(self, mock_validate): + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + path = self._make_file() + request = UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])]) + + self.vault.upload_files(request) + + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.vault_client.initialize_client_configuration.assert_called_once() + + def test_raises_for_invalid_request(self): + with self.assertRaises(SkyflowError): + self.vault.upload_files(UploadFilesRequest(records=[])) + self.vault_client.initialize_client_configuration.assert_not_called() + + def test_upload_via_file_path_success(self): + path = self._make_file(b"filedata") + self.files_api.with_raw_response.upload_files.return_value = self._ok_response(headers={"x-request-id": "req-u"}) + + response = self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])])) + + record = response.records[0] + self.assertEqual(record.skyflow_id, "sid1") + self.assertIsNone(record.error) + self.assertIsNone(record.request_id) + column = record.columns[0] + self.assertEqual(column.column, "c") + self.assertEqual(column.upload_status, FileUploadStatus.UPLOADED.value) + self.assertEqual(column.file_name, os.path.basename(path)) + self.assertFalse(hasattr(column, "signed_url")) + args, _ = self.vault_client.put_signed_url.call_args + self.assertEqual(args[0], "https://signed/url") + self.assertEqual(args[1], b"filedata") + _, kwargs = self.files_api.with_raw_response.upload_files.call_args + self.assertEqual(kwargs["records"][0].columns[0].file_name, os.path.basename(path)) + + def test_upload_via_base64_success(self): + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + encoded = base64.b64encode(b"xyz").decode() + + response = self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", base64=encoded, file_name="doc.pdf")])])) + + self.assertEqual(response.records[0].columns[0].upload_status, FileUploadStatus.UPLOADED.value) + args, _ = self.vault_client.put_signed_url.call_args + self.assertEqual(args[1], b"xyz") + + def test_upload_via_file_object_success(self): + path = self._make_file(b"objbytes", suffix=".bin") + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + + with open(path, "rb") as handle: + response = self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_object=handle)])])) + + args, _ = self.vault_client.put_signed_url.call_args + self.assertEqual(args[1], b"objbytes") + self.assertEqual(response.records[0].columns[0].file_name, os.path.basename(path)) + + def test_content_type_inferred_from_file_name(self): + path = self._make_file(suffix=".pdf") + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + + self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])])) + + args, _ = self.vault_client.put_signed_url.call_args + self.assertEqual(args[2], "application/pdf") + + def test_explicit_content_type_used(self): + path = self._make_file(suffix=".bin") + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + + self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path, content_type="application/custom")])])) + + args, _ = self.vault_client.put_signed_url.call_args + self.assertEqual(args[2], "application/custom") + + def test_per_record_error_marks_columns_skipped(self): + path = self._make_file() + self.files_api.with_raw_response.upload_files.return_value = fake_records_raw_response([ + FakeFileUploadResponseObject(skyflow_id=None, table_name="t1", data=None, error="bad record", http_code=400), + ], headers={"x-request-id": "req-e"}) + + response = self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])])) + + record = response.records[0] + self.assertEqual(record.error, "bad record") + self.assertEqual(record.request_id, "req-e") + self.assertEqual(record.columns[0].upload_status, FileUploadStatus.SKIPPED.value) + self.assertEqual(record.columns[0].error, "bad record") + self.vault_client.put_signed_url.assert_not_called() + + def test_put_failure_marks_column_failed(self): + self.vault_client.put_signed_url.return_value = SimpleNamespace(status_code=403) + path = self._make_file() + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + + response = self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])])) + + column = response.records[0].columns[0] + self.assertEqual(column.upload_status, FileUploadStatus.FAILED.value) + self.assertEqual(column.error, "PUT failed: 403") + + def test_put_exception_marks_column_failed(self): + self.vault_client.put_signed_url.side_effect = Exception("conn reset") + path = self._make_file() + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + + response = self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])])) + + column = response.records[0].columns[0] + self.assertEqual(column.upload_status, FileUploadStatus.FAILED.value) + self.assertIn("conn reset", column.error) + + def test_missing_signed_url_marks_column_skipped(self): + path = self._make_file() + self.files_api.with_raw_response.upload_files.return_value = fake_records_raw_response([ + FakeFileUploadResponseObject(skyflow_id="sid", table_name="t1", data={}, http_code=200), + ]) + + response = self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])])) + + self.assertEqual(response.records[0].columns[0].upload_status, FileUploadStatus.SKIPPED.value) + self.vault_client.put_signed_url.assert_not_called() + + def test_skyflow_id_sent_when_provided(self): + path = self._make_file() + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + + self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", skyflow_id="existing", columns=[UploadFilesRequestColumn(column="c", file_path=path)])])) + + _, kwargs = self.files_api.with_raw_response.upload_files.call_args + self.assertEqual(kwargs["records"][0].skyflow_id, "existing") + + def test_skyflow_id_omitted_when_absent(self): + path = self._make_file() + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + + self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])])) + + _, kwargs = self.files_api.with_raw_response.upload_files.call_args + self.assertIsNone(kwargs["records"][0].skyflow_id) + + def test_file_not_found_raises(self): + with self.assertRaises(SkyflowError): + self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path="/no/such/file.xyz")])])) + self.files_api.with_raw_response.upload_files.assert_not_called() + + def test_invalid_base64_raises(self): + with self.assertRaises(SkyflowError): + self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", base64="!!!not-base64!!!", file_name="x.bin")])])) + self.files_api.with_raw_response.upload_files.assert_not_called() + + def test_whole_call_error_raises(self): + path = self._make_file() + self.files_api.with_raw_response.upload_files.side_effect = ApiError( + status_code=401, headers={}, body={"error": "unauthorized"}) + with self.assertRaises(SkyflowError) as ctx: + self.vault.upload_files(UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])])) + self.assertEqual(ctx.exception.http_code, 401) + + def test_interceptor_adds_custom_header(self): + path = self._make_file() + self.files_api.with_raw_response.upload_files.return_value = self._ok_response() + + def interceptor(context): + context.add_header(CustomHeaderKey.REQUEST_ID_HEADER, "req-x") + + self.vault.upload_files( + UploadFilesRequest(records=[UploadFilesRequestRecord( + table_name="t1", columns=[UploadFilesRequestColumn(column="c", file_path=path)])]), + UploadFilesOptions(interceptor=interceptor), + ) + + _, kwargs = self.files_api.with_raw_response.upload_files.call_args + self.assertEqual(kwargs["request_options"]["additional_headers"]["x-request-id"], "req-x") + + +class TestVaultDeleteFiles(unittest.TestCase): + def setUp(self): + self.vault_client = Mock() + self.vault_client.get_vault_id.return_value = "vault123" + self.vault_client.get_logger.return_value = Mock() + self.vault_client.get_current_bearer_token.return_value = None + self.files_api = MagicMock() + self.vault_client.get_files_api.return_value = self.files_api + self.vault = VaultController(self.vault_client) + + @patch("skyflow.vault.controller._vault.validate_delete_files_request") + def test_validates_before_initializing_client(self, mock_validate): + self.files_api.with_raw_response.delete_files.return_value = fake_records_raw_response([]) + request = DeleteFilesRequest(records=[DeleteFilesRequestRecord(table_name="t1", columns=["c"], skyflow_id="id1")]) + + self.vault.delete_files(request) + + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.vault_client.initialize_client_configuration.assert_called_once() + + def test_raises_for_invalid_request(self): + with self.assertRaises(SkyflowError): + self.vault.delete_files(DeleteFilesRequest(records=[])) + self.vault_client.initialize_client_configuration.assert_not_called() + + def test_maps_by_skyflow_id(self): + self.files_api.with_raw_response.delete_files.return_value = fake_records_raw_response([]) + + self.vault.delete_files(DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c1", "c2"], skyflow_id="id1"), + ])) + + _, kwargs = self.files_api.with_raw_response.delete_files.call_args + self.assertEqual(kwargs["vault_id"], "vault123") + self.assertEqual(kwargs["records"][0].table_name, "t1") + self.assertEqual(kwargs["records"][0].columns, ["c1", "c2"]) + self.assertEqual(kwargs["records"][0].skyflow_id, "id1") + + def test_maps_by_unique_values(self): + self.files_api.with_raw_response.delete_files.return_value = fake_records_raw_response([]) + + self.vault.delete_files(DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c"], unique_values=[{"email": "a@b.com"}]), + ])) + + _, kwargs = self.files_api.with_raw_response.delete_files.call_args + self.assertEqual(kwargs["records"][0].unique_values[0].data, {"email": "a@b.com"}) + + def test_success_record_maps_column_statuses(self): + self.files_api.with_raw_response.delete_files.return_value = fake_records_raw_response([ + FakeFileDeleteResponseObject( + skyflow_id="id1", table_name="t1", + data={"c1": {"status": "DELETED"}, "c2": {"status": "DELETED"}}, http_code=200), + ], headers={"x-request-id": "req-d"}) + + response = self.vault.delete_files(DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c1", "c2"], skyflow_id="id1"), + ])) + + record = response.records[0] + self.assertEqual(record.skyflow_id, "id1") + self.assertIsNone(record.error) + self.assertIsNone(record.request_id) + self.assertEqual([(c.column, c.status) for c in record.columns], [("c1", "DELETED"), ("c2", "DELETED")]) + + def test_failed_record_has_none_columns(self): + self.files_api.with_raw_response.delete_files.return_value = fake_records_raw_response([ + FakeFileDeleteResponseObject(skyflow_id="bad", table_name="t1", data=None, error="invalid", http_code=404), + ], headers={"x-request-id": "req-f"}) + + response = self.vault.delete_files(DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c"], skyflow_id="bad"), + ])) + + record = response.records[0] + self.assertIsNone(record.columns) + self.assertEqual(record.error, "invalid") + self.assertEqual(record.http_code, 404) + self.assertEqual(record.request_id, "req-f") + + def test_transport_exception_raises_skyflow_error(self): + self.files_api.with_raw_response.delete_files.side_effect = Exception("network blip") + with self.assertRaises(SkyflowError) as ctx: + self.vault.delete_files(DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c"], skyflow_id="id1")])) + self.assertIn("network blip", ctx.exception.message) + + def test_api_error_with_per_record_body_returns_error_row(self): + self.files_api.with_raw_response.delete_files.side_effect = ApiError( + status_code=404, headers={"x-request-id": "req-3"}, + body={"records": [{"error": "invalid", "httpCode": 404}]}, + ) + + response = self.vault.delete_files(DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c"], skyflow_id="id1")])) + self.assertEqual(response.records[0].error, "invalid") + self.assertEqual(response.records[0].request_id, "req-3") + + +if __name__ == "__main__": + unittest.main() diff --git a/flowvault/tests/vault/data/test_new_data_classes.py b/flowvault/tests/vault/data/test_new_data_classes.py new file mode 100644 index 00000000..55d8fb58 --- /dev/null +++ b/flowvault/tests/vault/data/test_new_data_classes.py @@ -0,0 +1,149 @@ +import unittest + +from common.vault.data import ( + BaseQueryRequest, + BaseQueryResponse, + BaseGetTokensRequest, + BaseGetTokensResponse, + BaseUploadFilesRequest, + BaseUploadFilesResponse, + BaseDeleteFilesRequest, + BaseDeleteFilesResponse, +) +from skyflow.utils.enums import FileUploadStatus +from skyflow.vault.data import ( + QueryRequest, + QueryOptions, + QueryResponse, + QueryResponseRecord, + QueryResponseMetadata, + GetTokensRequest, + GetTokensRequestRecord, + GetTokensOptions, + GetTokensResponse, + GetTokensResponseRecord, + UploadFilesRequest, + UploadFilesRequestRecord, + UploadFilesRequestColumn, + UploadFilesOptions, + UploadFilesResponse, + UploadFilesResponseRecord, + UploadFilesColumnResult, + DeleteFilesRequest, + DeleteFilesRequestRecord, + DeleteFilesOptions, + DeleteFilesResponse, + DeleteFilesResponseRecord, + DeleteFilesColumnResult, +) + + +class TestQueryDataClasses(unittest.TestCase): + def test_request_extends_base(self): + request = QueryRequest(query="SELECT 1") + self.assertIsInstance(request, BaseQueryRequest) + self.assertEqual(request.query, "SELECT 1") + + def test_options_default_none(self): + self.assertIsNone(QueryOptions().interceptor) + + def test_record_and_metadata(self): + record = QueryResponseRecord(data={"a": 1}) + metadata = QueryResponseMetadata(columns=["a"]) + self.assertEqual(record.data, {"a": 1}) + self.assertEqual(metadata.columns, ["a"]) + self.assertIn("QueryResponseRecord", repr(record)) + self.assertIn("QueryResponseMetadata", str(metadata)) + + def test_response_extends_base_and_repr(self): + response = QueryResponse(records=[QueryResponseRecord(data={"a": 1})], + metadata=QueryResponseMetadata(columns=["a"]), request_id="rid") + self.assertIsInstance(response, BaseQueryResponse) + self.assertEqual(response.request_id, "rid") + self.assertIn("QueryResponse", repr(response)) + self.assertIn("request_id", str(response)) + + +class TestGetTokensDataClasses(unittest.TestCase): + def test_request_extends_base(self): + request = GetTokensRequest(records=[GetTokensRequestRecord(value="v", token_group_name="g")]) + self.assertIsInstance(request, BaseGetTokensRequest) + self.assertEqual(request.records[0].value, "v") + self.assertEqual(request.records[0].token_group_name, "g") + + def test_options_default_none(self): + self.assertIsNone(GetTokensOptions().interceptor) + + def test_response_and_record(self): + record = GetTokensResponseRecord(value="v", token_group_name="g", token="tok", http_code=200, error=None, request_id=None) + response = GetTokensResponse(records=[record]) + self.assertIsInstance(response, BaseGetTokensResponse) + self.assertEqual(response.records[0].token, "tok") + self.assertIn("GetTokensResponseRecord", repr(record)) + self.assertIn("GetTokensResponse", repr(response)) + + +class TestUploadFilesDataClasses(unittest.TestCase): + def test_request_extends_base(self): + column = UploadFilesRequestColumn(column="c", file_path="/tmp/x") + request = UploadFilesRequest(records=[UploadFilesRequestRecord(table_name="t1", columns=[column])]) + self.assertIsInstance(request, BaseUploadFilesRequest) + self.assertEqual(request.records[0].table_name, "t1") + self.assertIsNone(request.records[0].skyflow_id) + self.assertEqual(request.records[0].columns[0].column, "c") + + def test_column_accepts_each_source(self): + self.assertEqual(UploadFilesRequestColumn(column="c", file_path="/p").file_path, "/p") + self.assertEqual(UploadFilesRequestColumn(column="c", base64="b", file_name="n").base64, "b") + sentinel = object() + self.assertIs(UploadFilesRequestColumn(column="c", file_object=sentinel).file_object, sentinel) + + def test_options_default_none(self): + self.assertIsNone(UploadFilesOptions().interceptor) + + def test_response_record_and_column_result(self): + column_result = UploadFilesColumnResult(column="c", file_name="f.txt", + upload_status=FileUploadStatus.UPLOADED.value, error=None) + record = UploadFilesResponseRecord(skyflow_id="sid", table_name="t1", columns=[column_result], http_code=200) + response = UploadFilesResponse(records=[record]) + self.assertIsInstance(response, BaseUploadFilesResponse) + self.assertEqual(response.records[0].columns[0].upload_status, "UPLOADED") + self.assertFalse(hasattr(column_result, "signed_url")) + self.assertIn("UploadFilesColumnResult", repr(column_result)) + self.assertIn("UploadFilesResponseRecord", repr(record)) + self.assertIn("UploadFilesResponse", repr(response)) + + +class TestDeleteFilesDataClasses(unittest.TestCase): + def test_request_extends_base(self): + request = DeleteFilesRequest(records=[ + DeleteFilesRequestRecord(table_name="t1", columns=["c"], skyflow_id="id1"), + ]) + self.assertIsInstance(request, BaseDeleteFilesRequest) + self.assertEqual(request.records[0].columns, ["c"]) + self.assertEqual(request.records[0].skyflow_id, "id1") + self.assertIsNone(request.records[0].unique_values) + + def test_options_default_none(self): + self.assertIsNone(DeleteFilesOptions().interceptor) + + def test_response_record_and_column_result(self): + column_result = DeleteFilesColumnResult(column="c", status="DELETED") + record = DeleteFilesResponseRecord(skyflow_id="id1", table_name="t1", columns=[column_result], http_code=200) + response = DeleteFilesResponse(records=[record]) + self.assertIsInstance(response, BaseDeleteFilesResponse) + self.assertEqual(response.records[0].columns[0].status, "DELETED") + self.assertIn("DeleteFilesColumnResult", repr(column_result)) + self.assertIn("DeleteFilesResponseRecord", repr(record)) + self.assertIn("DeleteFilesResponse", str(response)) + + +class TestFileUploadStatusEnum(unittest.TestCase): + def test_values(self): + self.assertEqual(FileUploadStatus.UPLOADED.value, "UPLOADED") + self.assertEqual(FileUploadStatus.FAILED.value, "FAILED") + self.assertEqual(FileUploadStatus.SKIPPED.value, "SKIPPED") + + +if __name__ == "__main__": + unittest.main()