SK-3139: Implement query, get_tokens, upload_files, delete_files in flowvault

common/vault/data/__init__.py

@@ -1,2 +1,10 @@
 from ._base_insert_request import BaseInsertRequest
 from ._base_insert_response import BaseInsertResponse
+from ._base_query_request import BaseQueryRequest
+from ._base_query_response import BaseQueryResponse
+from ._base_get_tokens_request import BaseGetTokensRequest
+from ._base_get_tokens_response import BaseGetTokensResponse
+from ._base_upload_files_request import BaseUploadFilesRequest
+from ._base_upload_files_response import BaseUploadFilesResponse
+from ._base_delete_files_request import BaseDeleteFilesRequest
+from ._base_delete_files_response import BaseDeleteFilesResponse

common/vault/data/_base_delete_files_request.py

@@ -0,0 +1,3 @@
+class BaseDeleteFilesRequest:
+    def __init__(self, records):
+        self.records = records

common/vault/data/_base_delete_files_response.py

@@ -0,0 +1,9 @@
+class BaseDeleteFilesResponse:
+    def __init__(self, records=None):
+        self.records = records
+
+    def __repr__(self):
+        return f"{type(self).__name__}(records={self.records})"
+
+    def __str__(self):
+        return self.__repr__()

common/vault/data/_base_get_tokens_request.py

@@ -0,0 +1,3 @@
+class BaseGetTokensRequest:
+    def __init__(self, records):
+        self.records = records

common/vault/data/_base_get_tokens_response.py

@@ -0,0 +1,9 @@
+class BaseGetTokensResponse:
+    def __init__(self, records=None):
+        self.records = records
+
+    def __repr__(self):
+        return f"{type(self).__name__}(records={self.records})"
+
+    def __str__(self):
+        return self.__repr__()

common/vault/data/_base_query_request.py

@@ -0,0 +1,3 @@
+class BaseQueryRequest:
+    def __init__(self, query):
+        self.query = query

common/vault/data/_base_query_response.py

@@ -0,0 +1,9 @@
+class BaseQueryResponse:
+    def __init__(self, records=None):
+        self.records = records
+
+    def __repr__(self):
+        return f"{type(self).__name__}(records={self.records})"
+
+    def __str__(self):
+        return self.__repr__()

common/vault/data/_base_upload_files_request.py

@@ -0,0 +1,3 @@
+class BaseUploadFilesRequest:
+    def __init__(self, records):
+        self.records = records

common/vault/data/_base_upload_files_response.py

@@ -0,0 +1,9 @@
+class BaseUploadFilesResponse:
+    def __init__(self, records=None):
+        self.records = records
+
+    def __repr__(self):
+        return f"{type(self).__name__}(records={self.records})"
+
+    def __str__(self):
+        return self.__repr__()

flowvault/api-report/skyflow.api.json

@@ -34,16 +34,24 @@
   "skyflow.utils.enums.EnvUrls.PROD": "attr = 'skyvault.skyflowapis.com'",
   "skyflow.utils.enums.EnvUrls.SANDBOX": "attr = 'skyvault.skyflowapis-preview.com'",
   "skyflow.utils.enums.EnvUrls.STAGE": "attr = 'skyvault.skyflowapis.tech'",
+  "skyflow.utils.enums.FileUploadStatus": "class (Enum)",
+  "skyflow.utils.enums.FileUploadStatus.FAILED": "attr = 'FAILED'",
+  "skyflow.utils.enums.FileUploadStatus.SKIPPED": "attr = 'SKIPPED'",
+  "skyflow.utils.enums.FileUploadStatus.UPLOADED": "attr = 'UPLOADED'",
   "skyflow.utils.enums.UpsertType": "class (Enum)",
   "skyflow.utils.enums.UpsertType.REPLACE": "attr = 'REPLACE'",
   "skyflow.utils.enums.UpsertType.UPDATE": "attr = 'UPDATE'",
   "skyflow.vault.controller.VaultController": "class (BaseVaultController)",
   "skyflow.vault.controller.VaultController.__init__": "def (self, vault_client)",
   "skyflow.vault.controller.VaultController.delete": "def (self, request: DeleteRequest, options: DeleteOptions = None) -> DeleteResponse",
+  "skyflow.vault.controller.VaultController.delete_files": "def (self, request: DeleteFilesRequest, options: DeleteFilesOptions = None) -> DeleteFilesResponse",
   "skyflow.vault.controller.VaultController.detokenize": "def (self, request: DetokenizeRequest, options: DetokenizeOptions = None) -> DetokenizeResponse",
   "skyflow.vault.controller.VaultController.get": "def (self, request: GetRequest, options: GetOptions = None) -> GetResponse",
+  "skyflow.vault.controller.VaultController.get_tokens": "def (self, request: GetTokensRequest, options: GetTokensOptions = None) -> GetTokensResponse",
   "skyflow.vault.controller.VaultController.insert": "def (self, request: InsertRequest, options: InsertOptions = None) -> InsertResponse",
+  "skyflow.vault.controller.VaultController.query": "def (self, request: QueryRequest, options: QueryOptions = None) -> QueryResponse",
   "skyflow.vault.controller.VaultController.update": "def (self, request: UpdateRequest, options: UpdateOptions = None) -> UpdateResponse",
+  "skyflow.vault.controller.VaultController.upload_files": "def (self, request: UploadFilesRequest, options: UploadFilesOptions = None) -> UploadFilesResponse",
   "skyflow.vault.data.ColumnRedactions": "class ()",
   "skyflow.vault.data.ColumnRedactions.__init__": "def (self, column_name: str, redaction: str = None)",
   "skyflow.vault.data.ColumnRedactions.column_name": "attr",
@@ -52,6 +60,31 @@
   "skyflow.vault.data.CustomHeaderKey.REQUEST_ID_HEADER": "attr = 'x-request-id'",
   "skyflow.vault.data.CustomHeaderKey.SKYFLOW_ACCOUNT_ID": "attr = 'x-skyflow-account-id'",
   "skyflow.vault.data.CustomHeaderKey.SKYFLOW_ACCOUNT_NAME": "attr = 'x-skyflow-account-name'",
+  "skyflow.vault.data.DeleteFilesColumnResult": "class ()",
+  "skyflow.vault.data.DeleteFilesColumnResult.__init__": "def (self, column: str, status: str = None)",
+  "skyflow.vault.data.DeleteFilesColumnResult.column": "attr",
+  "skyflow.vault.data.DeleteFilesColumnResult.status": "attr",
+  "skyflow.vault.data.DeleteFilesOptions": "class ()",
+  "skyflow.vault.data.DeleteFilesOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)",
+  "skyflow.vault.data.DeleteFilesOptions.interceptor": "attr",
+  "skyflow.vault.data.DeleteFilesRequest": "class (BaseDeleteFilesRequest)",
+  "skyflow.vault.data.DeleteFilesRequest.__init__": "def (self, records: List[DeleteFilesRequestRecord])",
+  "skyflow.vault.data.DeleteFilesRequestRecord": "class ()",
+  "skyflow.vault.data.DeleteFilesRequestRecord.__init__": "def (self, table_name: str, columns: List[str], skyflow_id: str = None, unique_values: List[dict] = None)",
+  "skyflow.vault.data.DeleteFilesRequestRecord.columns": "attr",
+  "skyflow.vault.data.DeleteFilesRequestRecord.skyflow_id": "attr",
+  "skyflow.vault.data.DeleteFilesRequestRecord.table_name": "attr",
+  "skyflow.vault.data.DeleteFilesRequestRecord.unique_values": "attr",
+  "skyflow.vault.data.DeleteFilesResponse": "class (BaseDeleteFilesResponse)",
+  "skyflow.vault.data.DeleteFilesResponse.__init__": "def (self, records = None)",
+  "skyflow.vault.data.DeleteFilesResponseRecord": "class ()",
+  "skyflow.vault.data.DeleteFilesResponseRecord.__init__": "def (self, skyflow_id: str = None, table_name: str = None, columns = None, http_code: int = None, error: str = None, request_id: str = None)",
+  "skyflow.vault.data.DeleteFilesResponseRecord.columns": "attr",
+  "skyflow.vault.data.DeleteFilesResponseRecord.error": "attr",
+  "skyflow.vault.data.DeleteFilesResponseRecord.http_code": "attr",
+  "skyflow.vault.data.DeleteFilesResponseRecord.request_id": "attr",
+  "skyflow.vault.data.DeleteFilesResponseRecord.skyflow_id": "attr",
+  "skyflow.vault.data.DeleteFilesResponseRecord.table_name": "attr",
   "skyflow.vault.data.DeleteOptions": "class ()",
   "skyflow.vault.data.DeleteOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)",
   "skyflow.vault.data.DeleteOptions.interceptor": "attr",
@@ -125,6 +158,25 @@
   "skyflow.vault.data.GetResponseRecord.skyflow_id": "attr",
   "skyflow.vault.data.GetResponseRecord.table_name": "attr",
   "skyflow.vault.data.GetResponseRecord.tokens": "attr",
+  "skyflow.vault.data.GetTokensOptions": "class ()",
+  "skyflow.vault.data.GetTokensOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)",
+  "skyflow.vault.data.GetTokensOptions.interceptor": "attr",
+  "skyflow.vault.data.GetTokensRequest": "class (BaseGetTokensRequest)",
+  "skyflow.vault.data.GetTokensRequest.__init__": "def (self, records: List[GetTokensRequestRecord])",
+  "skyflow.vault.data.GetTokensRequestRecord": "class ()",
+  "skyflow.vault.data.GetTokensRequestRecord.__init__": "def (self, value, token_group_name: str)",
+  "skyflow.vault.data.GetTokensRequestRecord.token_group_name": "attr",
+  "skyflow.vault.data.GetTokensRequestRecord.value": "attr",
+  "skyflow.vault.data.GetTokensResponse": "class (BaseGetTokensResponse)",
+  "skyflow.vault.data.GetTokensResponse.__init__": "def (self, records = None)",
+  "skyflow.vault.data.GetTokensResponseRecord": "class ()",
+  "skyflow.vault.data.GetTokensResponseRecord.__init__": "def (self, value = None, token_group_name = None, token = None, http_code = None, error = None, request_id = None)",
+  "skyflow.vault.data.GetTokensResponseRecord.error": "attr",
+  "skyflow.vault.data.GetTokensResponseRecord.http_code": "attr",
+  "skyflow.vault.data.GetTokensResponseRecord.request_id": "attr",
+  "skyflow.vault.data.GetTokensResponseRecord.token": "attr",
+  "skyflow.vault.data.GetTokensResponseRecord.token_group_name": "attr",
+  "skyflow.vault.data.GetTokensResponseRecord.value": "attr",
   "skyflow.vault.data.InsertOptions": "class ()",
   "skyflow.vault.data.InsertOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)",
   "skyflow.vault.data.InsertOptions.interceptor": "attr",
@@ -152,6 +204,21 @@
   "skyflow.vault.data.InsertResponseRecord.skyflow_id": "attr",
   "skyflow.vault.data.InsertResponseRecord.table_name": "attr",
   "skyflow.vault.data.InsertResponseRecord.tokens": "attr",
+  "skyflow.vault.data.QueryOptions": "class ()",
+  "skyflow.vault.data.QueryOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)",
+  "skyflow.vault.data.QueryOptions.interceptor": "attr",
+  "skyflow.vault.data.QueryRequest": "class (BaseQueryRequest)",
+  "skyflow.vault.data.QueryRequest.__init__": "def (self, query: str)",
+  "skyflow.vault.data.QueryResponse": "class (BaseQueryResponse)",
+  "skyflow.vault.data.QueryResponse.__init__": "def (self, records = None, metadata = None, request_id = None)",
+  "skyflow.vault.data.QueryResponse.metadata": "attr",
+  "skyflow.vault.data.QueryResponse.request_id": "attr",
+  "skyflow.vault.data.QueryResponseMetadata": "class ()",
+  "skyflow.vault.data.QueryResponseMetadata.__init__": "def (self, columns: list = None)",
+  "skyflow.vault.data.QueryResponseMetadata.columns": "attr",
+  "skyflow.vault.data.QueryResponseRecord": "class ()",
+  "skyflow.vault.data.QueryResponseRecord.__init__": "def (self, data: dict = None)",
+  "skyflow.vault.data.QueryResponseRecord.data": "attr",
   "skyflow.vault.data.RequestContext": "class ()",
   "skyflow.vault.data.RequestContext.__init__": "def (self, operation, batch_index = NOT_BATCHED, total_batches = NOT_BATCHED)",
   "skyflow.vault.data.RequestContext.add_header": "def (self, key, value)",
@@ -195,6 +262,40 @@
   "skyflow.vault.data.UpdateResponseRecord.skyflow_id": "attr",
   "skyflow.vault.data.UpdateResponseRecord.table_name": "attr",
   "skyflow.vault.data.UpdateResponseRecord.tokens": "attr",
+  "skyflow.vault.data.UploadFilesColumnResult": "class ()",
+  "skyflow.vault.data.UploadFilesColumnResult.__init__": "def (self, column: str, file_name: str = None, upload_status: str = None, error: str = None)",
+  "skyflow.vault.data.UploadFilesColumnResult.column": "attr",
+  "skyflow.vault.data.UploadFilesColumnResult.error": "attr",
+  "skyflow.vault.data.UploadFilesColumnResult.file_name": "attr",
+  "skyflow.vault.data.UploadFilesColumnResult.upload_status": "attr",
+  "skyflow.vault.data.UploadFilesOptions": "class ()",
+  "skyflow.vault.data.UploadFilesOptions.__init__": "def (self, interceptor: Optional[Callable[[RequestContext], None]] = None)",
+  "skyflow.vault.data.UploadFilesOptions.interceptor": "attr",
+  "skyflow.vault.data.UploadFilesRequest": "class (BaseUploadFilesRequest)",
+  "skyflow.vault.data.UploadFilesRequest.__init__": "def (self, records: List[UploadFilesRequestRecord])",
+  "skyflow.vault.data.UploadFilesRequestColumn": "class ()",
+  "skyflow.vault.data.UploadFilesRequestColumn.__init__": "def (self, column: str, file_path: str = None, base64: str = None, file_object: BinaryIO = None, file_name: str = None, content_type: str = None)",
+  "skyflow.vault.data.UploadFilesRequestColumn.base64": "attr",
+  "skyflow.vault.data.UploadFilesRequestColumn.column": "attr",
+  "skyflow.vault.data.UploadFilesRequestColumn.content_type": "attr",
+  "skyflow.vault.data.UploadFilesRequestColumn.file_name": "attr",
+  "skyflow.vault.data.UploadFilesRequestColumn.file_object": "attr",
+  "skyflow.vault.data.UploadFilesRequestColumn.file_path": "attr",
+  "skyflow.vault.data.UploadFilesRequestRecord": "class ()",
+  "skyflow.vault.data.UploadFilesRequestRecord.__init__": "def (self, table_name: str, columns: List[UploadFilesRequestColumn], skyflow_id: str = None)",
+  "skyflow.vault.data.UploadFilesRequestRecord.columns": "attr",
+  "skyflow.vault.data.UploadFilesRequestRecord.skyflow_id": "attr",
+  "skyflow.vault.data.UploadFilesRequestRecord.table_name": "attr",
+  "skyflow.vault.data.UploadFilesResponse": "class (BaseUploadFilesResponse)",
+  "skyflow.vault.data.UploadFilesResponse.__init__": "def (self, records = None)",
+  "skyflow.vault.data.UploadFilesResponseRecord": "class ()",
+  "skyflow.vault.data.UploadFilesResponseRecord.__init__": "def (self, skyflow_id: str = None, table_name: str = None, columns = None, http_code: int = None, error: str = None, request_id: str = None)",
+  "skyflow.vault.data.UploadFilesResponseRecord.columns": "attr",
+  "skyflow.vault.data.UploadFilesResponseRecord.error": "attr",
+  "skyflow.vault.data.UploadFilesResponseRecord.http_code": "attr",
+  "skyflow.vault.data.UploadFilesResponseRecord.request_id": "attr",
+  "skyflow.vault.data.UploadFilesResponseRecord.skyflow_id": "attr",
+  "skyflow.vault.data.UploadFilesResponseRecord.table_name": "attr",
   "skyflow.vault.data.UpsertOptions": "class ()",
   "skyflow.vault.data.UpsertOptions.__init__": "def (self, unique_columns: list = None, update_type: UpsertType = None)",
   "skyflow.vault.data.UpsertOptions.unique_columns": "attr",

flowvault/samples/README.md

@@ -39,6 +39,10 @@ python flowvault/samples/vault_api/insert_records.py
 | [update_record.py](vault_api/update_record.py) | Update a record |
 | [delete_records.py](vault_api/delete_records.py) | Delete records |
 | [detokenize_records.py](vault_api/detokenize_records.py) | Detokenize tokens |
+| [query_records.py](vault_api/query_records.py) | Run a read-only SQL `SELECT` query |
+| [get_tokens.py](vault_api/get_tokens.py) | Look up existing tokens for values (deterministic token groups) |
+| [upload_files.py](vault_api/upload_files.py) | Upload files to file columns (`file_path` / `base64` / `file_object`) |
+| [delete_files.py](vault_api/delete_files.py) | Delete files from file columns (by `skyflow_id` or `unique_values`) |
 
 ## Custom headers & HTTP config
 

flowvault/samples/vault_api/delete_files.py

@@ -0,0 +1,55 @@
+from skyflow.error import SkyflowError
+from skyflow import Env
+from skyflow import Skyflow, LogLevel
+from skyflow.vault.data import DeleteFilesRequest, DeleteFilesRequestRecord
+
+
+def perform_delete_files():
+    try:
+        credentials = {
+            'path': '<PATH_TO_YOUR_CREDENTIALS_JSON>',
+        }
+
+        vault_config = {
+            'vault_id': '<YOUR_VAULT_ID>',
+            'cluster_id': '<YOUR_CLUSTER_ID>',
+            'env': Env.PROD,
+            'credentials': credentials,
+        }
+
+        skyflow_client = (
+            Skyflow.builder()
+            .add_vault_config(vault_config)
+            .set_log_level(LogLevel.ERROR)
+            .build()
+        )
+
+        # Each record sets exactly ONE of skyflow_id or unique_values to target the record(s).
+        delete_files_request = DeleteFilesRequest(
+            records=[
+                DeleteFilesRequestRecord(
+                    table_name='<SENSITIVE_DATA_TABLE>',
+                    columns=['<FILE_COLUMN>'],
+                    skyflow_id='<SKYFLOW_ID>',
+                ),
+            ],
+        )
+
+        response = skyflow_client.vault(vault_config.get('vault_id')).delete_files(delete_files_request)
+
+        # response.records: list of DeleteFilesResponseRecord, one per resolved record, in order.
+        # Each has .skyflow_id, .table_name, .columns, .http_code, .error, .request_id.
+        # Each column result has .column and .status ("DELETED"); .columns is None on a failed record.
+        print('Records: ', response.records)
+
+    except SkyflowError as error:
+        print('Skyflow Specific Error: ', {
+            'code': error.http_code,
+            'message': error.message,
+            'details': error.details,
+        })
+    except Exception as error:
+        print('Unexpected Error:', error)
+
+
+perform_delete_files()

flowvault/samples/vault_api/get_tokens.py

@@ -0,0 +1,51 @@
+from skyflow.error import SkyflowError
+from skyflow import Env
+from skyflow import Skyflow, LogLevel
+from skyflow.vault.data import GetTokensRequest, GetTokensRequestRecord
+
+
+def perform_get_tokens():
+    try:
+        credentials = {
+            'path': '<PATH_TO_YOUR_CREDENTIALS_JSON>',
+        }
+
+        vault_config = {
+            'vault_id': '<YOUR_VAULT_ID>',
+            'cluster_id': '<YOUR_CLUSTER_ID>',
+            'env': Env.PROD,
+            'credentials': credentials,
+        }
+
+        skyflow_client = (
+            Skyflow.builder()
+            .add_vault_config(vault_config)
+            .set_log_level(LogLevel.ERROR)
+            .build()
+        )
+
+        # One record per input value; only deterministic token groups are supported.
+        get_tokens_request = GetTokensRequest(
+            records=[
+                GetTokensRequestRecord(value='john@example.com', token_group_name='<TOKEN_GROUP_NAME>'),
+                GetTokensRequestRecord(value='jane@example.com', token_group_name='<TOKEN_GROUP_NAME>'),
+            ],
+        )
+
+        response = skyflow_client.vault(vault_config.get('vault_id')).get_tokens(get_tokens_request)
+
+        # response.records: list of GetTokensResponseRecord, one per input, in order.
+        # Each has .value, .token_group_name, .token, .http_code, .error, .request_id.
+        print('Records: ', response.records)
+
+    except SkyflowError as error:
+        print('Skyflow Specific Error: ', {
+            'code': error.http_code,
+            'message': error.message,
+            'details': error.details,
+        })
+    except Exception as error:
+        print('Unexpected Error:', error)
+
+
+perform_get_tokens()

flowvault/samples/vault_api/query_records.py

@@ -0,0 +1,50 @@
+from skyflow.error import SkyflowError
+from skyflow import Env
+from skyflow import Skyflow, LogLevel
+from skyflow.vault.data import QueryRequest
+
+
+def perform_query():
+    try:
+        credentials = {
+            'path': '<PATH_TO_YOUR_CREDENTIALS_JSON>',
+        }
+
+        vault_config = {
+            'vault_id': '<YOUR_VAULT_ID>',
+            'cluster_id': '<YOUR_CLUSTER_ID>',
+            'env': Env.PROD,
+            'credentials': credentials,
+        }
+
+        skyflow_client = (
+            Skyflow.builder()
+            .add_vault_config(vault_config)
+            .set_log_level(LogLevel.ERROR)
+            .build()
+        )
+
+        # Only SELECT is supported; a call returns at most 25 records -- page with OFFSET.
+        query_request = QueryRequest(
+            query="SELECT name, email FROM <SENSITIVE_DATA_TABLE> LIMIT 25",
+        )
+
+        response = skyflow_client.vault(vault_config.get('vault_id')).query(query_request)
+
+        # response.records: list of QueryResponseRecord, each with .data (a dict of column -> value).
+        # response.metadata.columns lists the returned columns; response.request_id is always set.
+        print('Request ID: ', response.request_id)
+        print('Columns: ', response.metadata.columns if response.metadata else None)
+        print('Records: ', response.records)
+
+    except SkyflowError as error:
+        print('Skyflow Specific Error: ', {
+            'code': error.http_code,
+            'message': error.message,
+            'details': error.details,
+        })
+    except Exception as error:
+        print('Unexpected Error:', error)
+
+
+perform_query()