From 32649a6870cc49e56ffdf8cbf82d0370641e6839 Mon Sep 17 00:00:00 2001 From: Xander Date: Mon, 14 Sep 2026 11:45:12 +0100 Subject: [PATCH 1/5] tmp --- mkdocs/docs/index.md | 1 + pyiceberg/encryption/ciphers.py | 120 +++++++++++++++++++++++++++ pyiceberg/transforms.py | 33 +++----- pyiceberg/utils/lazy_import.py | 54 ++++++++++++ pyproject.toml | 1 + tests/encryption/test_ciphers.py | 137 +++++++++++++++++++++++++++++++ tests/utils/test_lazy_import.py | 40 +++++++++ uv.lock | 6 +- 8 files changed, 368 insertions(+), 24 deletions(-) create mode 100644 pyiceberg/encryption/ciphers.py create mode 100644 pyiceberg/utils/lazy_import.py create mode 100644 tests/encryption/test_ciphers.py create mode 100644 tests/utils/test_lazy_import.py diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index a37f3be8b0..f8d376be67 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -66,6 +66,7 @@ You can mix and match optional dependencies depending on your needs: | hf | Support for Hugging Face Hub | | gcp-auth | Support for Google Cloud authentication | | entra-auth | Support for Azure Entra authentication | +| encryption | Support for table encryption | You either need to install `s3fs`, `adlfs`, `gcsfs`, or `pyarrow` to be able to fetch files from an object store. diff --git a/pyiceberg/encryption/ciphers.py b/pyiceberg/encryption/ciphers.py new file mode 100644 index 0000000000..60a3507dd2 --- /dev/null +++ b/pyiceberg/encryption/ciphers.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""AES-GCM primitives for table encryption.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from enum import IntEnum +from typing import TYPE_CHECKING + +from pyiceberg.utils.lazy_import import not_installed + +if TYPE_CHECKING: + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + +class AesKeySize(IntEnum): + """The supported AES key sizes, in bits.""" + + BITS_128 = 128 + BITS_192 = 192 + BITS_256 = 256 + + @property + def key_length(self) -> int: + """Return the key length in bytes.""" + return self // 8 + + @classmethod + def from_key_length(cls, key_length: int) -> AesKeySize: + """Return the key size for a key of `key_length` bytes.""" + try: + return cls(key_length * 8) + except ValueError as e: + raise ValueError(f"Unsupported key length: {key_length} (must be 16, 24 or 32)") from e + + +@dataclass(frozen=True) +class SecureKey: + """An AES key of a length the spec allows, kept out of reprs and tracebacks.""" + + key: bytes = field(repr=False) + + def __post_init__(self) -> None: + """Reject keys that are not a supported AES key length.""" + AesKeySize.from_key_length(len(self.key)) + + @property + def key_size(self) -> AesKeySize: + """Return the size of this key.""" + return AesKeySize.from_key_length(len(self.key)) + + @classmethod + def generate(cls, key_size: AesKeySize = AesKeySize.BITS_128) -> SecureKey: + """Generate a new key of `key_size`.""" + return cls(os.urandom(key_size.key_length)) + + +class AesGcmCipher: + """Encrypts and decrypts using AES-GCM. + + Ciphertext is laid out as `nonce || ciphertext || tag`, matching Java and iceberg-rust. + """ + + NONCE_LENGTH = 12 + TAG_LENGTH = 16 + + def __init__(self, key: SecureKey) -> None: + try: + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + except ImportError: + raise not_installed("cryptography", extras_name="encryption") from None + + self._aes_gcm: AESGCM = AESGCM(key.key) + self._invalid_tag: type[InvalidTag] = InvalidTag + + def encrypt(self, plaintext: bytes, aad: bytes | None = None) -> bytes: + """Encrypt `plaintext`, authenticating `aad` alongside it. + + Args: + plaintext (bytes): The data to encrypt. + aad (bytes | None): Additional data to authenticate but not encrypt. + """ + nonce = os.urandom(self.NONCE_LENGTH) + return nonce + self._aes_gcm.encrypt(nonce, plaintext, aad) + + def decrypt(self, ciphertext: bytes, aad: bytes | None = None) -> bytes: + """Decrypt `ciphertext`, verifying `aad` alongside it. + + Args: + ciphertext (bytes): The data to decrypt, as returned by `encrypt`. + aad (bytes | None): The additional data that was authenticated on encryption. + """ + if len(ciphertext) < self.NONCE_LENGTH + self.TAG_LENGTH: + raise ValueError( + f"Ciphertext too short: expected at least {self.NONCE_LENGTH + self.TAG_LENGTH} bytes, got {len(ciphertext)}" + ) + + nonce, encrypted = ciphertext[: self.NONCE_LENGTH], ciphertext[self.NONCE_LENGTH :] + try: + return self._aes_gcm.decrypt(nonce, encrypted, aad) + except self._invalid_tag as e: + raise ValueError("AES-GCM decryption failed") from e diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 5e0027a829..54c01d9bed 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -17,9 +17,7 @@ import base64 import datetime as py_datetime -import importlib import struct -import types from abc import ABC, abstractmethod from collections.abc import Callable from enum import IntEnum @@ -31,7 +29,6 @@ import mmh3 from pydantic import Field, PositiveInt, PrivateAttr -from pyiceberg.exceptions import NotInstalledError from pyiceberg.expressions import ( BoundEqualTo, BoundGreaterThan, @@ -88,6 +85,7 @@ ) from pyiceberg.utils import datetime from pyiceberg.utils.decimal import decimal_to_bytes, truncate_decimal +from pyiceberg.utils.lazy_import import try_import from pyiceberg.utils.parsing import ParseNumberFromBrackets from pyiceberg.utils.singleton import Singleton @@ -112,17 +110,6 @@ TRUNCATE_PARSER = ParseNumberFromBrackets(TRUNCATE) -def _try_import(module_name: str, extras_name: str | None = None) -> types.ModuleType: - try: - return importlib.import_module(module_name) - except ImportError: - if extras_name: - msg = f'{module_name} needs to be installed. pip install "pyiceberg[{extras_name}]"' - else: - msg = f"{module_name} needs to be installed." - raise NotInstalledError(msg) from None - - def _transform_literal(func: Callable[[Any], Any], lit: Literal[L]) -> Literal[L]: """Small helper to upwrap the value from the literal, and wrap it again.""" return literal(func(lit.value)) @@ -395,7 +382,7 @@ def __repr__(self) -> str: return f"BucketTransform(num_buckets={self._num_buckets})" def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": - pyiceberg_core_transform = _try_import("pyiceberg_core", extras_name="pyiceberg-core").transform + pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.bucket, self._num_buckets) @@ -509,8 +496,8 @@ def __repr__(self) -> str: return "YearTransform()" def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": - pa = _try_import("pyarrow") - pyiceberg_core_transform = _try_import("pyiceberg_core", extras_name="pyiceberg-core").transform + pa = try_import("pyarrow") + pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.year, expected_type=pa.int32()) @@ -569,8 +556,8 @@ def __repr__(self) -> str: return "MonthTransform()" def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": - pa = _try_import("pyarrow") - pyiceberg_core_transform = _try_import("pyiceberg_core", extras_name="pyiceberg-core").transform + pa = try_import("pyarrow") + pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.month, expected_type=pa.int32()) @@ -638,8 +625,8 @@ def __repr__(self) -> str: return "DayTransform()" def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": - pa = _try_import("pyarrow", extras_name="pyarrow") - pyiceberg_core_transform = _try_import("pyiceberg_core", extras_name="pyiceberg-core").transform + pa = try_import("pyarrow", extras_name="pyarrow") + pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.day, expected_type=pa.int32()) @@ -691,7 +678,7 @@ def __repr__(self) -> str: return "HourTransform()" def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": - pyiceberg_core_transform = _try_import("pyiceberg_core", extras_name="pyiceberg-core").transform + pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.hour) @@ -918,7 +905,7 @@ def __repr__(self) -> str: return f"TruncateTransform(width={self._width})" def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]": - pyiceberg_core_transform = _try_import("pyiceberg_core", extras_name="pyiceberg-core").transform + pyiceberg_core_transform = try_import("pyiceberg_core", extras_name="pyiceberg-core").transform return _pyiceberg_transform_wrapper(pyiceberg_core_transform.truncate, self._width) diff --git a/pyiceberg/utils/lazy_import.py b/pyiceberg/utils/lazy_import.py new file mode 100644 index 0000000000..2df9b3e741 --- /dev/null +++ b/pyiceberg/utils/lazy_import.py @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Helpers for importing modules that ship in an optional extra.""" + +from __future__ import annotations + +import importlib +import types + +from pyiceberg.exceptions import NotInstalledError + + +def not_installed(module_name: str, extras_name: str | None = None) -> NotInstalledError: + """Return the error to raise when `module_name` is unavailable. + + Use this when the import has to be spelled out to keep the imported names statically typed; + prefer `try_import` otherwise. + + Args: + module_name (str): The module that could not be imported. + extras_name (str | None): The pyiceberg extra that provides it, if any. + """ + if extras_name: + msg = f'{module_name} needs to be installed. pip install "pyiceberg[{extras_name}]"' + else: + msg = f"{module_name} needs to be installed." + return NotInstalledError(msg) + + +def try_import(module_name: str, extras_name: str | None = None) -> types.ModuleType: + """Import `module_name`, raising `NotInstalledError` with an install hint when it is missing. + + Args: + module_name (str): The module to import. + extras_name (str | None): The pyiceberg extra that provides it, if any. + """ + try: + return importlib.import_module(module_name) + except ImportError: + raise not_installed(module_name, extras_name) from None diff --git a/pyproject.toml b/pyproject.toml index cf37a6b8f8..716020a24c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,6 +101,7 @@ datafusion = ["datafusion>=53,<54"] gcp-auth = ["google-auth>=2.4.0"] entra-auth = ["azure-identity>=1.25.1"] geoarrow = ["geoarrow-pyarrow>=0.2.0"] +encryption = ["cryptography>=42.0.0"] [dependency-groups] dev = [ diff --git a/tests/encryption/test_ciphers.py b/tests/encryption/test_ciphers.py new file mode 100644 index 0000000000..1e694a60f5 --- /dev/null +++ b/tests/encryption/test_ciphers.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +from pytest_mock import MockFixture + +from pyiceberg.encryption.ciphers import AesGcmCipher, AesKeySize, SecureKey +from pyiceberg.exceptions import NotInstalledError + +AES128_KEY = b"0123456789012345" +PLAINTEXT = b"the quick brown fox" + + +@pytest.mark.parametrize( + "key_length, key_size", + [(16, AesKeySize.BITS_128), (24, AesKeySize.BITS_192), (32, AesKeySize.BITS_256)], +) +def test_key_size_from_key_length(key_length: int, key_size: AesKeySize) -> None: + assert AesKeySize.from_key_length(key_length) == key_size + assert key_size.key_length == key_length + + +@pytest.mark.parametrize("key_length", [0, 4, 15, 20, 33]) +def test_key_size_rejects_invalid_key_length(key_length: int) -> None: + with pytest.raises(ValueError, match=f"Unsupported key length: {key_length}"): + AesKeySize.from_key_length(key_length) + + +@pytest.mark.parametrize("key_length", [0, 4, 15, 20, 33]) +def test_secure_key_rejects_invalid_key_length(key_length: int) -> None: + with pytest.raises(ValueError, match="Unsupported key length"): + SecureKey(bytes(key_length)) + + +@pytest.mark.parametrize("key_size", list(AesKeySize)) +def test_secure_key_generate(key_size: AesKeySize) -> None: + key = SecureKey.generate(key_size) + + assert len(key.key) == key_size.key_length + assert key.key_size == key_size + assert SecureKey.generate(key_size) != key + + +def test_secure_key_repr_redacts_key() -> None: + key = SecureKey(AES128_KEY) + + assert repr(key) == "SecureKey()" + assert repr(AES128_KEY) not in repr(key) + + +@pytest.mark.parametrize("key_size", list(AesKeySize)) +@pytest.mark.parametrize("aad", [None, b"", b"aad"]) +def test_encrypt_decrypt_round_trip(key_size: AesKeySize, aad: bytes | None) -> None: + cipher = AesGcmCipher(SecureKey.generate(key_size)) + + ciphertext = cipher.encrypt(PLAINTEXT, aad) + + assert ciphertext != PLAINTEXT + assert cipher.decrypt(ciphertext, aad) == PLAINTEXT + + +def test_encrypt_empty_plaintext() -> None: + cipher = AesGcmCipher(SecureKey(AES128_KEY)) + + assert cipher.decrypt(cipher.encrypt(b"")) == b"" + + +def test_ciphertext_layout() -> None: + cipher = AesGcmCipher(SecureKey(AES128_KEY)) + + ciphertext = cipher.encrypt(PLAINTEXT) + + assert len(ciphertext) == AesGcmCipher.NONCE_LENGTH + len(PLAINTEXT) + AesGcmCipher.TAG_LENGTH + + +def test_nonce_is_not_reused() -> None: + cipher = AesGcmCipher(SecureKey(AES128_KEY)) + + first, second = cipher.encrypt(PLAINTEXT), cipher.encrypt(PLAINTEXT) + + assert first[: AesGcmCipher.NONCE_LENGTH] != second[: AesGcmCipher.NONCE_LENGTH] + assert first != second + + +def test_decrypt_with_wrong_key() -> None: + ciphertext = AesGcmCipher(SecureKey(AES128_KEY)).encrypt(PLAINTEXT) + + with pytest.raises(ValueError, match="AES-GCM decryption failed"): + AesGcmCipher(SecureKey(b"5432109876543210")).decrypt(ciphertext) + + +def test_decrypt_with_mismatched_aad() -> None: + cipher = AesGcmCipher(SecureKey(AES128_KEY)) + + ciphertext = cipher.encrypt(PLAINTEXT, b"aad") + + with pytest.raises(ValueError, match="AES-GCM decryption failed"): + cipher.decrypt(ciphertext, b"other aad") + + +def test_decrypt_tampered_ciphertext() -> None: + cipher = AesGcmCipher(SecureKey(AES128_KEY)) + + ciphertext = bytearray(cipher.encrypt(PLAINTEXT)) + ciphertext[-1] ^= 0xFF + + with pytest.raises(ValueError, match="AES-GCM decryption failed"): + cipher.decrypt(bytes(ciphertext)) + + +@pytest.mark.parametrize("length", [0, 1, 27]) +def test_decrypt_ciphertext_too_short(length: int) -> None: + cipher = AesGcmCipher(SecureKey(AES128_KEY)) + + with pytest.raises(ValueError, match=f"Ciphertext too short: expected at least 28 bytes, got {length}"): + cipher.decrypt(bytes(length)) + + +def test_cipher_without_cryptography_installed_raises_not_installed_error(mocker: MockFixture) -> None: + mocker.patch.dict("sys.modules", {"cryptography.hazmat.primitives.ciphers.aead": None}) + + with pytest.raises(NotInstalledError, match=r"pyiceberg\[encryption\]"): + AesGcmCipher(SecureKey(AES128_KEY)) diff --git a/tests/utils/test_lazy_import.py b/tests/utils/test_lazy_import.py new file mode 100644 index 0000000000..657fb76f8b --- /dev/null +++ b/tests/utils/test_lazy_import.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import importlib + +import pytest + +from pyiceberg.exceptions import NotInstalledError +from pyiceberg.utils.lazy_import import not_installed, try_import + + +def test_try_import_returns_the_module() -> None: + assert try_import("importlib") is importlib + + +def test_try_import_missing_module_with_extras() -> None: + with pytest.raises(NotInstalledError, match=r'nonexistent needs to be installed. pip install "pyiceberg\[some-extra\]"'): + try_import("nonexistent", extras_name="some-extra") + + +def test_try_import_missing_module_without_extras() -> None: + with pytest.raises(NotInstalledError, match="nonexistent needs to be installed."): + try_import("nonexistent") + + +def test_not_installed_returns_rather_than_raises() -> None: + assert isinstance(not_installed("nonexistent", extras_name="some-extra"), NotInstalledError) diff --git a/uv.lock b/uv.lock index 200f2ad2e6..d78aa2f74a 100644 --- a/uv.lock +++ b/uv.lock @@ -4932,6 +4932,9 @@ duckdb = [ dynamodb = [ { name = "boto3" }, ] +encryption = [ + { name = "cryptography" }, +] entra-auth = [ { name = "azure-identity" }, ] @@ -5049,6 +5052,7 @@ requires-dist = [ { name = "boto3", marker = "extra == 'rest-sigv4'", specifier = ">=1.24.59" }, { name = "cachetools", specifier = ">=5.5" }, { name = "click", specifier = ">=7.1.1" }, + { name = "cryptography", marker = "extra == 'encryption'", specifier = ">=42.0.0" }, { name = "daft", marker = "extra == 'daft'", specifier = ">=0.7.10" }, { name = "datafusion", marker = "extra == 'datafusion'", specifier = ">=53,<54" }, { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=0.5.0" }, @@ -5086,7 +5090,7 @@ requires-dist = [ { name = "thrift-sasl", marker = "extra == 'hive-kerberos'", specifier = ">=0.4.3" }, { name = "zstandard", specifier = ">=0.13.0" }, ] -provides-extras = ["pyarrow", "pandas", "duckdb", "ray", "bodo", "daft", "polars", "snappy", "hive", "hive-kerberos", "s3fs", "glue", "adlfs", "dynamodb", "bigquery", "sql-postgres", "sql-sqlite", "gcsfs", "rest-sigv4", "hf", "pyiceberg-core", "datafusion", "gcp-auth", "entra-auth", "geoarrow"] +provides-extras = ["pyarrow", "pandas", "duckdb", "ray", "bodo", "daft", "polars", "snappy", "hive", "hive-kerberos", "s3fs", "glue", "adlfs", "dynamodb", "bigquery", "sql-postgres", "sql-sqlite", "gcsfs", "rest-sigv4", "hf", "pyiceberg-core", "datafusion", "gcp-auth", "entra-auth", "geoarrow", "encryption"] [package.metadata.requires-dev] dev = [ From a91af754f4bd85649ae345be2a3115e8de28aaf0 Mon Sep 17 00:00:00 2001 From: Xander Date: Mon, 14 Sep 2026 18:10:43 +0100 Subject: [PATCH 2/5] nit --- pyiceberg/encryption/ciphers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyiceberg/encryption/ciphers.py b/pyiceberg/encryption/ciphers.py index 60a3507dd2..3d3cac51b0 100644 --- a/pyiceberg/encryption/ciphers.py +++ b/pyiceberg/encryption/ciphers.py @@ -40,7 +40,7 @@ class AesKeySize(IntEnum): @property def key_length(self) -> int: """Return the key length in bytes.""" - return self // 8 + return self.value // 8 @classmethod def from_key_length(cls, key_length: int) -> AesKeySize: From 6858d2d1cf6872632f531d8e92f197a1902ead63 Mon Sep 17 00:00:00 2001 From: Xander Date: Mon, 14 Sep 2026 21:02:14 +0000 Subject: [PATCH 3/5] test(encryption): add AES-GCM known-answer regression tests The existing cipher tests all round-trip through the same code, so a symmetric change to the nonce || ciphertext || tag layout would pass them while breaking interoperability with the Java and iceberg-rust clients. Pin encryption and decryption against fixed GCM-spec vectors covering AES-128 and AES-256, with and without AAD. --- tests/encryption/test_ciphers.py | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/encryption/test_ciphers.py b/tests/encryption/test_ciphers.py index 1e694a60f5..f02685e895 100644 --- a/tests/encryption/test_ciphers.py +++ b/tests/encryption/test_ciphers.py @@ -24,6 +24,40 @@ AES128_KEY = b"0123456789012345" PLAINTEXT = b"the quick brown fox" +# Known-answer vectors from McGrew & Viega, "The Galois/Counter Mode of Operation +# (GCM)", shared with the NIST GCM validation suite and the Java and iceberg-rust +# test suites. They pin the `nonce || ciphertext || tag` layout against changes +# that stay self-consistent on round trip but break cross-client interoperability. +GCM_TEST_VECTORS = [ + pytest.param( + "feffe9928665731c6d6a8f9467308308", + "cafebabefacedbaddecaf888", + "d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b391aafd255", + "", + "42831ec2217774244b7221b784d0d49ce3aa212f2c02a4e035c17e2329aca12e21d514b25466931c7d8f6a5aac84aa051ba30b396a0aac973d58e091473f5985", + "4d5c2af327cd64a62cf35abd2ba6fab4", + id="aes128-no-aad", + ), + pytest.param( + "feffe9928665731c6d6a8f9467308308", + "cafebabefacedbaddecaf888", + "d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39", + "feedfacedeadbeeffeedfacedeadbeefabaddad2", + "42831ec2217774244b7221b784d0d49ce3aa212f2c02a4e035c17e2329aca12e21d514b25466931c7d8f6a5aac84aa051ba30b396a0aac973d58e091", + "5bc94fbc3221a5db94fae95ae7121a47", + id="aes128-with-aad", + ), + pytest.param( + "feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308", + "cafebabefacedbaddecaf888", + "d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39", + "feedfacedeadbeeffeedfacedeadbeefabaddad2", + "522dc1f099567d07f47f37a32a84427d643a8cdcbfe5c0c97598a2bd2555d1aa8cb08e48590dbb3da7b08b1056828838c5f61e6393ba7a0abcc9f662", + "76fc6ece0f4e1768cddf8853bb2d551b", + id="aes256-with-aad", + ), +] + @pytest.mark.parametrize( "key_length, key_size", @@ -73,6 +107,24 @@ def test_encrypt_decrypt_round_trip(key_size: AesKeySize, aad: bytes | None) -> assert cipher.decrypt(ciphertext, aad) == PLAINTEXT +@pytest.mark.parametrize("key, nonce, plaintext, aad, ciphertext, tag", GCM_TEST_VECTORS) +def test_decrypt_known_answer(key: str, nonce: str, plaintext: str, aad: str, ciphertext: str, tag: str) -> None: + cipher = AesGcmCipher(SecureKey(bytes.fromhex(key))) + stored = bytes.fromhex(nonce + ciphertext + tag) + + assert cipher.decrypt(stored, bytes.fromhex(aad) or None) == bytes.fromhex(plaintext) + + +@pytest.mark.parametrize("key, nonce, plaintext, aad, ciphertext, tag", GCM_TEST_VECTORS) +def test_encrypt_known_answer( + monkeypatch: pytest.MonkeyPatch, key: str, nonce: str, plaintext: str, aad: str, ciphertext: str, tag: str +) -> None: + monkeypatch.setattr("pyiceberg.encryption.ciphers.os.urandom", lambda _: bytes.fromhex(nonce)) + cipher = AesGcmCipher(SecureKey(bytes.fromhex(key))) + + assert cipher.encrypt(bytes.fromhex(plaintext), bytes.fromhex(aad) or None) == bytes.fromhex(nonce + ciphertext + tag) + + def test_encrypt_empty_plaintext() -> None: cipher = AesGcmCipher(SecureKey(AES128_KEY)) From 762143959761e65ee85fb194618c42ccdc79326b Mon Sep 17 00:00:00 2001 From: Xander Date: Mon, 14 Sep 2026 21:05:58 +0000 Subject: [PATCH 4/5] fix(encryption): align AES-GCM decryption error with Java Report the same cause as the Java client when the GCM tag check fails, so the message reads consistently across implementations and makes clear the failure is bad input rather than a retryable system error. --- pyiceberg/encryption/ciphers.py | 2 +- tests/encryption/test_ciphers.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyiceberg/encryption/ciphers.py b/pyiceberg/encryption/ciphers.py index 3d3cac51b0..4fc8089dcd 100644 --- a/pyiceberg/encryption/ciphers.py +++ b/pyiceberg/encryption/ciphers.py @@ -117,4 +117,4 @@ def decrypt(self, ciphertext: bytes, aad: bytes | None = None) -> bytes: try: return self._aes_gcm.decrypt(nonce, encrypted, aad) except self._invalid_tag as e: - raise ValueError("AES-GCM decryption failed") from e + raise ValueError("GCM tag check failed. Possible reasons: wrong decryption key; or corrupt/tampered data") from e diff --git a/tests/encryption/test_ciphers.py b/tests/encryption/test_ciphers.py index f02685e895..dc3dc7193d 100644 --- a/tests/encryption/test_ciphers.py +++ b/tests/encryption/test_ciphers.py @@ -151,7 +151,7 @@ def test_nonce_is_not_reused() -> None: def test_decrypt_with_wrong_key() -> None: ciphertext = AesGcmCipher(SecureKey(AES128_KEY)).encrypt(PLAINTEXT) - with pytest.raises(ValueError, match="AES-GCM decryption failed"): + with pytest.raises(ValueError, match="wrong decryption key; or corrupt/tampered data"): AesGcmCipher(SecureKey(b"5432109876543210")).decrypt(ciphertext) @@ -160,7 +160,7 @@ def test_decrypt_with_mismatched_aad() -> None: ciphertext = cipher.encrypt(PLAINTEXT, b"aad") - with pytest.raises(ValueError, match="AES-GCM decryption failed"): + with pytest.raises(ValueError, match="wrong decryption key; or corrupt/tampered data"): cipher.decrypt(ciphertext, b"other aad") @@ -170,7 +170,7 @@ def test_decrypt_tampered_ciphertext() -> None: ciphertext = bytearray(cipher.encrypt(PLAINTEXT)) ciphertext[-1] ^= 0xFF - with pytest.raises(ValueError, match="AES-GCM decryption failed"): + with pytest.raises(ValueError, match="wrong decryption key; or corrupt/tampered data"): cipher.decrypt(bytes(ciphertext)) From b9f3bbe57e11c1c820d64d462ac20efb99973033 Mon Sep 17 00:00:00 2001 From: Xander Date: Mon, 14 Sep 2026 21:16:17 +0000 Subject: [PATCH 5/5] refactor(encryption): import cryptography through try_import Import the cryptography modules with try_import rather than a spelled-out try/except, and drop not_installed now that nothing calls it. The TYPE_CHECKING block already keeps AESGCM and InvalidTag statically typed, so lazy_import is back to a single entry point. --- pyiceberg/encryption/ciphers.py | 13 +++++-------- pyiceberg/utils/lazy_import.py | 23 +++++------------------ tests/utils/test_lazy_import.py | 6 +----- 3 files changed, 11 insertions(+), 31 deletions(-) diff --git a/pyiceberg/encryption/ciphers.py b/pyiceberg/encryption/ciphers.py index 4fc8089dcd..078d30ec5c 100644 --- a/pyiceberg/encryption/ciphers.py +++ b/pyiceberg/encryption/ciphers.py @@ -23,7 +23,7 @@ from enum import IntEnum from typing import TYPE_CHECKING -from pyiceberg.utils.lazy_import import not_installed +from pyiceberg.utils.lazy_import import try_import if TYPE_CHECKING: from cryptography.exceptions import InvalidTag @@ -82,14 +82,11 @@ class AesGcmCipher: TAG_LENGTH = 16 def __init__(self, key: SecureKey) -> None: - try: - from cryptography.exceptions import InvalidTag - from cryptography.hazmat.primitives.ciphers.aead import AESGCM - except ImportError: - raise not_installed("cryptography", extras_name="encryption") from None + aead = try_import("cryptography.hazmat.primitives.ciphers.aead", extras_name="encryption") + exceptions = try_import("cryptography.exceptions", extras_name="encryption") - self._aes_gcm: AESGCM = AESGCM(key.key) - self._invalid_tag: type[InvalidTag] = InvalidTag + self._aes_gcm: AESGCM = aead.AESGCM(key.key) + self._invalid_tag: type[InvalidTag] = exceptions.InvalidTag def encrypt(self, plaintext: bytes, aad: bytes | None = None) -> bytes: """Encrypt `plaintext`, authenticating `aad` alongside it. diff --git a/pyiceberg/utils/lazy_import.py b/pyiceberg/utils/lazy_import.py index 2df9b3e741..8f6d7496c0 100644 --- a/pyiceberg/utils/lazy_import.py +++ b/pyiceberg/utils/lazy_import.py @@ -24,23 +24,6 @@ from pyiceberg.exceptions import NotInstalledError -def not_installed(module_name: str, extras_name: str | None = None) -> NotInstalledError: - """Return the error to raise when `module_name` is unavailable. - - Use this when the import has to be spelled out to keep the imported names statically typed; - prefer `try_import` otherwise. - - Args: - module_name (str): The module that could not be imported. - extras_name (str | None): The pyiceberg extra that provides it, if any. - """ - if extras_name: - msg = f'{module_name} needs to be installed. pip install "pyiceberg[{extras_name}]"' - else: - msg = f"{module_name} needs to be installed." - return NotInstalledError(msg) - - def try_import(module_name: str, extras_name: str | None = None) -> types.ModuleType: """Import `module_name`, raising `NotInstalledError` with an install hint when it is missing. @@ -51,4 +34,8 @@ def try_import(module_name: str, extras_name: str | None = None) -> types.Module try: return importlib.import_module(module_name) except ImportError: - raise not_installed(module_name, extras_name) from None + if extras_name: + msg = f'{module_name} needs to be installed. pip install "pyiceberg[{extras_name}]"' + else: + msg = f"{module_name} needs to be installed." + raise NotInstalledError(msg) from None diff --git a/tests/utils/test_lazy_import.py b/tests/utils/test_lazy_import.py index 657fb76f8b..d68da5423f 100644 --- a/tests/utils/test_lazy_import.py +++ b/tests/utils/test_lazy_import.py @@ -19,7 +19,7 @@ import pytest from pyiceberg.exceptions import NotInstalledError -from pyiceberg.utils.lazy_import import not_installed, try_import +from pyiceberg.utils.lazy_import import try_import def test_try_import_returns_the_module() -> None: @@ -34,7 +34,3 @@ def test_try_import_missing_module_with_extras() -> None: def test_try_import_missing_module_without_extras() -> None: with pytest.raises(NotInstalledError, match="nonexistent needs to be installed."): try_import("nonexistent") - - -def test_not_installed_returns_rather_than_raises() -> None: - assert isinstance(not_installed("nonexistent", extras_name="some-extra"), NotInstalledError)