From 04c1a0525be92d29c644cd02accdf0259b44a4ae Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 13 Nov 2026 02:48:13 -0800 Subject: [PATCH] feat(guest): migrate the data disk and env across KMS key epochs A 0.6.4 KMS answers a GetAppKey without an epoch selector with the keys of the current epochs. Once its root or app epoch is rotated, a 0.5.x guest gets keys that open neither its data disk nor its encrypted env. Backport the guest side of the 0.6.4 key epoch migration: - GetAppKey takes optional root/app epochs and returns the served epochs and the SealedEnv key. Historical requests pin the root CA this boot established and do not extend RTMR3. - Replace the luks2 crate header check with the 0.6.4 parser, which also accepts the one dstack-kms token recording the epochs of the keyslot, and re-key the data disk to the current epochs in crash-safe steps. An epoch-0 disk stays untouched, so older images still open it. - Open an env sealed with HPKE (SealedEnv) or, failing the current key, a legacy env with the keys of older epochs, then re-seal it to the current epochs and hand it to the VMM with UpdateSealedEnv, which also closes the env update window before applications start. A VMM without the method has no window; the 0.5.x VMM accepts and ignores the update. The 0.6.4 initializing label and the application-facing historical GetKey are not part of this backport. Ported from v0.6.4: dstack-util/src/system_setup/{luks,sealed_env}.rs, the env migration in system_setup.rs, crypto.rs sealed env functions, host_api.rs update_sealed_env and the http-client StatusError. Signed-off-by: Kevin Wang --- Cargo.lock | 588 +++++------ Cargo.toml | 3 +- dstack-types/src/lib.rs | 23 + dstack-types/src/shared_filenames.rs | 1 + dstack-util/Cargo.toml | 3 +- dstack-util/src/crypto.rs | 118 +++ dstack-util/src/host_api.rs | 104 +- dstack-util/src/main.rs | 1 + dstack-util/src/system_setup.rs | 996 +++++++++++------- dstack-util/src/system_setup/luks.rs | 969 +++++++++++++++++ dstack-util/src/system_setup/sealed_env.rs | 57 + .../tests/fixtures/sealed-env-javascript.json | 18 + .../tests/fixtures/sealed-env-python.json | 18 + guest-agent/src/rpc_service.rs | 1 + host-api/proto/host_api.proto | 11 + host-api/src/client.rs | 1 + http-client/src/prpc.rs | 57 +- kms/rpc/proto/kms_rpc.proto | 9 + kms/src/main_service.rs | 3 + vmm/src/host_api_service.rs | 15 +- 20 files changed, 2265 insertions(+), 731 deletions(-) create mode 100644 dstack-util/src/system_setup/luks.rs create mode 100644 dstack-util/src/system_setup/sealed_env.rs create mode 100644 dstack-util/tests/fixtures/sealed-env-javascript.json create mode 100644 dstack-util/tests/fixtures/sealed-env-python.json diff --git a/Cargo.lock b/Cargo.lock index a80a4eb39..f629508c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,20 +14,18 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] [[package]] -name = "aes" -version = "0.7.5" +name = "aead" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ - "cfg-if", - "cipher 0.3.0", - "cpufeatures 0.2.17", - "opaque-debug", + "crypto-common 0.2.2", + "inout 0.2.6", ] [[package]] @@ -41,20 +39,45 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "aes" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1" +dependencies = [ + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.0", +] + [[package]] name = "aes-gcm" version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ - "aead", + "aead 0.5.2", "aes 0.8.4", "cipher 0.4.4", - "ctr", - "ghash", + "ctr 0.9.2", + "ghash 0.5.1", "subtle", ] +[[package]] +name = "aes-gcm" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +dependencies = [ + "aead 0.6.1", + "aes 0.9.4", + "cipher 0.5.2", + "ctr 0.10.1", + "ctutils", + "ghash 0.6.0", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -305,7 +328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e93e50f64a77ad9c5470bf2ad0ca02f228da70c792a8f06634801e202579f35e" dependencies = [ "alloy-rlp-derive", - "arrayvec 0.7.6", + "arrayvec", "bytes", ] @@ -472,7 +495,7 @@ checksum = "4d7fd448ab0a017de542de1dcca7a58e7019fe0e7a34ed3f9543ebddf6aceffa" dependencies = [ "alloy-primitives", "alloy-rlp", - "arrayvec 0.7.6", + "arrayvec", "derive_more 2.1.1", "nybbles", "serde", @@ -538,7 +561,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -549,7 +572,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -615,7 +638,7 @@ dependencies = [ "ark-ff-macros 0.5.0", "ark-serialize 0.5.0", "ark-std 0.5.0", - "arrayvec 0.7.6", + "arrayvec", "digest 0.10.7", "educe", "itertools 0.13.0", @@ -721,7 +744,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" dependencies = [ "ark-std 0.5.0", - "arrayvec 0.7.6", + "arrayvec", "digest 0.10.7", "num-bigint", ] @@ -768,12 +791,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "arrayvec" version = "0.7.6" @@ -999,15 +1016,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bincode" version = "2.0.1" @@ -1092,17 +1100,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "blake2b_simd" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" -dependencies = [ - "arrayref", - "arrayvec 0.5.2", - "constant_time_eq 0.1.5", -] - [[package]] name = "blake3" version = "1.8.3" @@ -1110,29 +1107,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" dependencies = [ "arrayref", - "arrayvec 0.7.6", + "arrayvec", "cc", "cfg-if", - "constant_time_eq 0.4.2", + "constant_time_eq", "cpufeatures 0.2.17", ] [[package]] name = "block-buffer" -version = "0.9.0" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -1402,7 +1399,7 @@ checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -1421,22 +1418,24 @@ dependencies = [ [[package]] name = "cipher" -version = "0.3.0" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "generic-array", + "crypto-common 0.1.7", + "inout 0.1.4", + "zeroize", ] [[package]] name = "cipher" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ - "crypto-common", - "inout", - "zeroize", + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.6", ] [[package]] @@ -1514,6 +1513,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.4" @@ -1570,12 +1575,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1646,6 +1645,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1718,31 +1723,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crossterm" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c36c10130df424b2f3552fcc2ddcd9b28a27b1e54b358b45874f88d1ca6888c" -dependencies = [ - "bitflags 1.3.2", - "crossterm_winapi", - "lazy_static", - "libc", - "mio 0.7.14", - "parking_lot 0.11.2", - "signal-hook 0.1.17", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0da8964ace4d3e4a044fd027919b2237000b24315a37c916f61809f1ff2140b9" -dependencies = [ - "winapi", -] - [[package]] name = "crunchy" version = "0.2.4" @@ -1773,13 +1753,12 @@ dependencies = [ ] [[package]] -name = "crypto-mac" -version = "0.11.0" +name = "crypto-common" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "subtle", + "hybrid-array", ] [[package]] @@ -1811,6 +1790,24 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher 0.5.2", +] + +[[package]] +name = "ctutils" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f" +dependencies = [ + "cmov", +] + [[package]] name = "cuckoofilter" version = "0.5.0" @@ -1832,7 +1829,22 @@ dependencies = [ "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", - "fiat-crypto", + "fiat-crypto 0.2.9", + "rustc_version 0.4.1", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "curve25519-dalek-derive", + "fiat-crypto 0.3.0", "rustc_version 0.4.1", "subtle", "zeroize", @@ -1930,7 +1942,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.12", + "parking_lot_core", ] [[package]] @@ -2165,10 +2177,21 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "6.0.0" @@ -2187,7 +2210,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2463,7 +2486,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", - "x25519-dalek", + "x25519-dalek 2.0.1", "x509-parser", "yasna", ] @@ -2582,7 +2605,7 @@ dependencies = [ name = "dstack-util" version = "0.5.13" dependencies = [ - "aes-gcm", + "aes-gcm 0.10.3", "anyhow", "base64 0.22.1", "bollard", @@ -2590,7 +2613,7 @@ dependencies = [ "cert-client", "clap", "cmd_lib", - "curve25519-dalek", + "curve25519-dalek 4.1.3", "dcap-qvl", "dstack-attest", "dstack-gateway-rpc", @@ -2603,9 +2626,9 @@ dependencies = [ "hex", "hex_fmt", "host-api", + "hpke", "k256", "libc", - "luks2", "parity-scale-codec", "ra-rpc", "ra-tls", @@ -2617,6 +2640,7 @@ dependencies = [ "serde", "serde-human-bytes", "serde_json", + "serde_with", "sha2 0.10.9", "sha3", "sodiumbox", @@ -2626,7 +2650,7 @@ dependencies = [ "toml", "tracing", "tracing-subscriber", - "x25519-dalek", + "x25519-dalek 2.0.1", "x509-parser", "yaml-rust2", ] @@ -2770,7 +2794,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ - "curve25519-dalek", + "curve25519-dalek 4.1.3", "ed25519", "rand_core 0.6.4", "serde", @@ -2934,7 +2958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2975,7 +2999,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139834ddba373bbdd213dffe02c8d110508dcf1726c2be27e8d1f7d7e1856418" dependencies = [ - "arrayvec 0.7.6", + "arrayvec", "auto_impl", "bytes", ] @@ -2986,7 +3010,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce8dba4714ef14b8274c371879b175aa55b16b30f269663f19d576f380018dc4" dependencies = [ - "arrayvec 0.7.6", + "arrayvec", "auto_impl", "bytes", ] @@ -3019,6 +3043,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "figment" version = "0.10.19" @@ -3322,7 +3352,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "rand_core 0.10.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -3344,7 +3374,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" dependencies = [ "opaque-debug", - "polyval", + "polyval 0.6.2", +] + +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval 0.7.3", ] [[package]] @@ -3512,7 +3551,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" dependencies = [ - "arrayvec 0.7.6", + "arrayvec", ] [[package]] @@ -3588,7 +3627,7 @@ dependencies = [ "ipconfig", "lru-cache", "once_cell", - "parking_lot 0.12.5", + "parking_lot", "rand 0.8.5", "resolv-conf", "smallvec", @@ -3609,7 +3648,7 @@ dependencies = [ "ipconfig", "moka", "once_cell", - "parking_lot 0.12.5", + "parking_lot", "rand 0.9.2", "resolv-conf", "smallvec", @@ -3628,13 +3667,12 @@ dependencies = [ ] [[package]] -name = "hmac" -version = "0.11.0" +name = "hkdf" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "crypto-mac", - "digest 0.9.0", + "hmac 0.13.0", ] [[package]] @@ -3646,6 +3684,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "home" version = "0.5.12" @@ -3668,6 +3715,25 @@ dependencies = [ "serde_json", ] +[[package]] +name = "hpke" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f512e325a1a71bf3273fb9b7fc9e5d96dac136fb4a95d0e6637001b12ea3e09" +dependencies = [ + "aead 0.6.1", + "aes-gcm 0.11.1", + "curve25519-dalek 5.0.0", + "getrandom 0.4.2", + "hkdf 0.13.0", + "hybrid-array", + "rand_core 0.10.1", + "sha2 0.11.1", + "subtle", + "x25519-dalek 3.0.0", + "zeroize", +] + [[package]] name = "http" version = "1.4.0" @@ -3763,6 +3829,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.8.1" @@ -4073,6 +4148,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "inout" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a580c13f0ffc335ef42dd3f6b1a41b726051c5e1656d9a3aa6fddb3c5dfbbf9" +dependencies = [ + "hybrid-array", +] + [[package]] name = "insta" version = "1.46.3" @@ -4085,15 +4169,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - [[package]] name = "instant-acme" version = "0.7.2" @@ -4177,7 +4252,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4460,28 +4535,6 @@ dependencies = [ "insta", ] -[[package]] -name = "luks2" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2044d8bd5489b199890c3dbf38d4c8f50f3a5a38833986808b14e2367fe267fa" -dependencies = [ - "aes 0.7.5", - "base64 0.13.1", - "bincode 1.3.3", - "crossterm", - "hmac 0.11.0", - "pbkdf2", - "rust-argon2", - "secrecy", - "serde", - "serde-big-array", - "serde_json", - "sha2 0.9.9", - "thiserror 1.0.69", - "xts-mode", -] - [[package]] name = "macro-string" version = "0.1.4" @@ -4583,19 +4636,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "mio" -version = "0.7.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" -dependencies = [ - "libc", - "log", - "miow", - "ntapi 0.3.7", - "winapi", -] - [[package]] name = "mio" version = "1.1.1" @@ -4608,15 +4648,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", -] - [[package]] name = "moka" version = "0.12.14" @@ -4627,7 +4658,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "equivalent", - "parking_lot 0.12.5", + "parking_lot", "portable-atomic", "smallvec", "tagptr", @@ -4772,7 +4803,7 @@ dependencies = [ "kqueue", "libc", "log", - "mio 1.1.1", + "mio", "notify-types", "walkdir", "windows-sys 0.60.2", @@ -4787,15 +4818,6 @@ dependencies = [ "bitflags 2.11.0", ] -[[package]] -name = "ntapi" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" -dependencies = [ - "winapi", -] - [[package]] name = "ntapi" version = "0.4.3" @@ -4811,7 +4833,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5007,7 +5029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -5062,7 +5084,7 @@ version = "3.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "799781ae679d79a948e13d4824a40970bfa500058d245760dd857301059810fa" dependencies = [ - "arrayvec 0.7.6", + "arrayvec", "bitvec", "byte-slice-cast", "const_format", @@ -5084,17 +5106,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -5102,21 +5113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core 0.9.12", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -5156,15 +5153,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "pbkdf2" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95f5254224e617595d2cc3cc73ff0a5eaf2637519e25f03388154e9378b6ffa" -dependencies = [ - "crypto-mac", -] - [[package]] name = "pear" version = "0.2.9" @@ -5358,7 +5346,7 @@ checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", ] [[package]] @@ -5370,7 +5358,18 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "opaque-debug", - "universal-hash", + "universal-hash 0.5.1", +] + +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures 0.3.0", + "universal-hash 0.6.1", ] [[package]] @@ -5816,7 +5815,7 @@ dependencies = [ "fs-err", "hex", "hex_fmt", - "hkdf", + "hkdf 0.12.4", "or-panic", "p256", "parity-scale-codec", @@ -5885,7 +5884,7 @@ checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" dependencies = [ "chacha20", "getrandom 0.4.2", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -5948,9 +5947,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rand_hc" @@ -5993,15 +5992,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -6257,7 +6247,7 @@ dependencies = [ "memchr", "multer", "num_cpus", - "parking_lot 0.12.5", + "parking_lot", "pin-project-lite", "rand 0.9.2", "ref-cast", @@ -6399,18 +6389,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" -[[package]] -name = "rust-argon2" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" -dependencies = [ - "base64 0.13.1", - "blake2b_simd", - "constant_time_eq 0.1.5", - "crossbeam-utils", -] - [[package]] name = "rustc-hash" version = "2.1.1" @@ -6473,7 +6451,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6790,10 +6768,10 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9fcb6c2e176e86ec703e22560d99d65a5ee9056ae45a08e13e84ebf796296f" dependencies = [ - "aead", + "aead 0.5.2", "arrayref", - "arrayvec 0.7.6", - "curve25519-dalek", + "arrayvec", + "curve25519-dalek 4.1.3", "getrandom_or_panic", "merlin", "rand_core 0.6.4", @@ -6860,15 +6838,6 @@ dependencies = [ "cc", ] -[[package]] -name = "secrecy" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" -dependencies = [ - "zeroize", -] - [[package]] name = "security-framework" version = "3.7.0" @@ -6926,15 +6895,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-big-array" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd31f59f6fe2b0c055371bb2f16d7f0aa7d8881676c04a55b1596d1a17cd10a4" -dependencies = [ - "serde", -] - [[package]] name = "serde-duration" version = "0.5.13" @@ -7105,26 +7065,24 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.9" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "block-buffer 0.9.0", "cfg-if", "cpufeatures 0.2.17", - "digest 0.9.0", - "opaque-debug", + "digest 0.10.7", ] [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -7181,18 +7139,7 @@ checksum = "47106eded3c154e70176fc83df9737335c94ce22f821c32d17ed1db1f83badb1" dependencies = [ "libc", "os_pipe", - "signal-hook 0.3.18", -] - -[[package]] -name = "signal-hook" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e31d442c16f047a671b5a71e2161d6e68814012b7f5379d269ebd915fac2729" -dependencies = [ - "libc", - "mio 0.7.14", - "signal-hook-registry", + "signal-hook", ] [[package]] @@ -7317,7 +7264,7 @@ dependencies = [ "or-panic", "rand_core 0.6.4", "salsa20", - "x25519-dalek", + "x25519-dalek 2.0.1", "xsalsa20poly1305", ] @@ -7516,7 +7463,7 @@ checksum = "3c3ffa3e4ff2b324a57f7aeb3c349656c7b127c3c189520251a648102a92496e" dependencies = [ "libc", "memchr", - "ntapi 0.4.3", + "ntapi", "objc2-core-foundation", "objc2-io-kit", "windows 0.61.3", @@ -7607,7 +7554,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7763,8 +7710,8 @@ checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", - "mio 1.1.1", - "parking_lot 0.12.5", + "mio", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.6.2", @@ -7999,7 +7946,7 @@ dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "parking_lot 0.12.5", + "parking_lot", "regex-automata", "sharded-slab", "smallvec", @@ -8027,9 +7974,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ubyte" @@ -8104,10 +8051,20 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -8368,7 +8325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9b73bc556dfdb7ef33617a9d477b803198db43ea3df25463efaf43d4986fe8" dependencies = [ "anyhow", - "bincode 2.0.1", + "bincode", "chrono", "crc32fast", "dashmap", @@ -8465,7 +8422,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -8996,12 +8953,23 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ - "curve25519-dalek", + "curve25519-dalek 4.1.3", "rand_core 0.6.4", "serde", "zeroize", ] +[[package]] +name = "x25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" +dependencies = [ + "curve25519-dalek 5.0.0", + "rand_core 0.10.1", + "zeroize", +] + [[package]] name = "x509-cert" version = "0.2.5" @@ -9047,23 +9015,13 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02a6dad357567f81cd78ee75f7c61f1b30bb2fe4390be8fb7c69e2ac8dffb6c7" dependencies = [ - "aead", + "aead 0.5.2", "poly1305", "salsa20", "subtle", "zeroize", ] -[[package]] -name = "xts-mode" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a099a2f21d48275314733f85bc43b6c6213b66394233aaea573fc7a520dcd9" -dependencies = [ - "byteorder", - "cipher 0.3.0", -] - [[package]] name = "yaml-rust2" version = "0.10.4" diff --git a/Cargo.toml b/Cargo.toml index d4b708040..db28bd37e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,7 @@ scale = { version = "3.7.4", package = "parity-scale-codec", features = [ ] } serde = { version = "1.0.228", features = ["derive"], default-features = false } serde-human-bytes = "0.1.2" +serde_with = "3.14.0" rmp-serde = "1.3.1" serde_json = { version = "1.0.140", default-features = false } serde_ini = "0.2.0" @@ -195,6 +196,7 @@ sha3 = "0.10.8" blake2 = "0.10.6" tokio-rustls = { version = "0.26.2", features = ["ring"] } x25519-dalek = { version = "2.0.1", features = ["static_secrets"] } +hpke = { version = "0.15.1", default-features = false, features = ["alloc", "getrandom", "x25519", "aes"] } k256 = "0.13.4" ed25519-dalek = { version = "2.2.0", features = ["rand_core"] } # Additional RustCrypto dependencies for sealed box @@ -244,7 +246,6 @@ smallvec = "1.14.0" cmd_lib = "1.9.5" yaml-rust2 = "0.10.4" -luks2 = "0.5.0" scopeguard = "1.2.0" tar = "0.4" proxy-protocol = "0.5.0" diff --git a/dstack-types/src/lib.rs b/dstack-types/src/lib.rs index fc493125b..56aed014b 100644 --- a/dstack-types/src/lib.rs +++ b/dstack-types/src/lib.rs @@ -234,6 +234,8 @@ pub struct AppKeys { pub disk_crypt_key: Vec, #[serde(with = "hex_bytes", default)] pub env_crypt_key: Vec, + #[serde(with = "hex_bytes", default)] + pub sealed_env_key: Vec, #[serde(with = "hex_bytes")] pub k256_key: Vec, #[serde(with = "hex_bytes")] @@ -264,9 +266,30 @@ pub enum KeyProvider { pubkey: Vec, tmp_ca_key: String, tmp_ca_cert: String, + /// KMS root key epoch the keys were derived from. + #[serde(default)] + root_epoch: u32, + #[serde(default)] + app_epoch: u32, }, } +impl AppKeys { + pub fn app_epoch(&self) -> u32 { + match self.key_provider { + KeyProvider::Kms { app_epoch, .. } => app_epoch, + _ => 0, + } + } + + pub fn root_epoch(&self) -> u32 { + match self.key_provider { + KeyProvider::Kms { root_epoch, .. } => root_epoch, + _ => 0, + } + } +} + impl KeyProvider { pub fn kind(&self) -> KeyProviderKind { match self { diff --git a/dstack-types/src/shared_filenames.rs b/dstack-types/src/shared_filenames.rs index 5c3ef8282..55ca15eff 100644 --- a/dstack-types/src/shared_filenames.rs +++ b/dstack-types/src/shared_filenames.rs @@ -7,6 +7,7 @@ pub const APP_KEYS: &str = ".appkeys.json"; pub const SYS_CONFIG: &str = ".sys-config.json"; pub const USER_CONFIG: &str = ".user-config"; pub const ENCRYPTED_ENV: &str = ".encrypted-env"; +pub const SEALED_ENV: &str = ".sealed-env"; pub const DECRYPTED_ENV: &str = ".decrypted-env"; pub const DECRYPTED_ENV_JSON: &str = ".decrypted-env.json"; pub const INSTANCE_INFO: &str = ".instance_info"; diff --git a/dstack-util/Cargo.toml b/dstack-util/Cargo.toml index 1bd1fb1bd..34a2a74db 100644 --- a/dstack-util/Cargo.toml +++ b/dstack-util/Cargo.toml @@ -23,6 +23,8 @@ scale = { workspace = true, features = ["derive"] } schnorrkel.workspace = true serde.workspace = true serde-human-bytes.workspace = true +serde_with.workspace = true +hpke.workspace = true serde_json.workspace = true sha2.workspace = true tokio = { workspace = true, features = ["full"] } @@ -50,7 +52,6 @@ yaml-rust2.workspace = true bollard.workspace = true sodiumbox.workspace = true libc.workspace = true -luks2.workspace = true scopeguard.workspace = true tempfile.workspace = true ez-hash.workspace = true diff --git a/dstack-util/src/crypto.rs b/dstack-util/src/crypto.rs index ee1dc555f..6b2f194bd 100644 --- a/dstack-util/src/crypto.rs +++ b/dstack-util/src/crypto.rs @@ -41,6 +41,71 @@ pub fn dh_decrypt(secret: [u8; 32], ciphertext: &[u8]) -> Result> { .map_err(|e| anyhow!("Decryption failed: {}", e)) } +/// Encrypt to `remote_public_key` in the format [`dh_decrypt`] reads: +/// ephemeral public key, IV, then the AES-GCM ciphertext. +#[cfg(test)] +pub fn dh_encrypt(remote_public_key: [u8; 32], plaintext: &[u8]) -> Result> { + let ephemeral_secret = StaticSecret::random_from_rng(rand::thread_rng()); + let shared_secret = ephemeral_secret + .diffie_hellman(&PublicKey::from(remote_public_key)) + .to_bytes(); + let cipher = Aes256Gcm::new_from_slice(&shared_secret) + .map_err(|e| anyhow!("failed to create cipher: {e}"))?; + let iv: [u8; 12] = rand::random(); + let ciphertext = cipher + .encrypt(Nonce::::from_slice(&iv), plaintext) + .map_err(|e| anyhow!("encryption failed: {e}"))?; + Ok([ + PublicKey::from(&ephemeral_secret).as_bytes(), + &iv[..], + &ciphertext, + ] + .concat()) +} + +/// Encrypt a sealed environment with HPKE DHKEM(X25519, HKDF-SHA256) and AES-256-GCM. +pub fn sealed_env_encrypt(public_key: &[u8], info: &[u8], plaintext: &[u8]) -> Result> { + use hpke::{ + aead::AesGcm256, kdf::HkdfSha256, kem::X25519HkdfSha256, Deserializable, OpModeS, + Serializable, + }; + let public_key = ::PublicKey::from_bytes(public_key) + .map_err(|_| anyhow!("invalid sealed env public key"))?; + let (enc, ciphertext) = hpke::single_shot_seal::( + &OpModeS::Base, + &public_key, + info, + plaintext, + &[], + ) + .map_err(|e| anyhow!("sealed env encryption failed: {e}"))?; + Ok([enc.to_bytes().as_ref(), ciphertext.as_slice()].concat()) +} + +/// Decrypt a sealed environment HPKE payload. +pub fn sealed_env_decrypt(secret: &[u8], info: &[u8], payload: &[u8]) -> Result> { + use hpke::{aead::AesGcm256, kdf::HkdfSha256, kem::X25519HkdfSha256, Deserializable, OpModeR}; + let enc = payload + .get(..32) + .ok_or_else(|| anyhow!("sealed env payload is missing encapsulated key"))?; + let ciphertext = payload + .get(32..) + .ok_or_else(|| anyhow!("sealed env payload is missing ciphertext"))?; + let secret = ::PrivateKey::from_bytes(secret) + .map_err(|_| anyhow!("invalid sealed env private key"))?; + let enc = ::EncappedKey::from_bytes(enc) + .map_err(|_| anyhow!("invalid sealed env encapsulated key"))?; + hpke::single_shot_open::( + &OpModeR::Base, + &secret, + &enc, + info, + ciphertext, + &[], + ) + .map_err(|e| anyhow!("sealed env decryption failed: {e}")) +} + #[cfg(test)] mod tests { use super::*; @@ -86,4 +151,57 @@ mod tests { let decrypted_str = String::from_utf8(decrypted).unwrap(); assert_eq!(decrypted_str, "[{\"key\":\"\",\"value\":\"\"}]"); } + + #[test] + fn dh_encrypt_round_trips_through_dh_decrypt() { + let secret = [7u8; 32]; + let public = PublicKey::from(&StaticSecret::from(secret)).to_bytes(); + let ciphertext = dh_encrypt(public, b"env").unwrap(); + assert_eq!(dh_decrypt(secret, &ciphertext).unwrap(), b"env"); + assert!(dh_decrypt([8u8; 32], &ciphertext).is_err()); + } + + #[test] + fn opens_sdk_sealed_envs() { + use base64::Engine as _; + for source in [ + include_str!("../tests/fixtures/sealed-env-python.json"), + include_str!("../tests/fixtures/sealed-env-javascript.json"), + ] { + let fixture: serde_json::Value = serde_json::from_str(source).unwrap(); + let envelope = &fixture["sealed_env"]; + let decode = |name: &str| { + base64::engine::general_purpose::STANDARD + .decode(envelope[name].as_str().unwrap()) + .unwrap() + }; + let payload = [decode("enc"), decode("ciphertext")].concat(); + let plaintext = sealed_env_decrypt( + &hex::decode(fixture["private_key"].as_str().unwrap()).unwrap(), + b"dstack-sealed-env-v1:3:0", + &payload, + ) + .unwrap(); + let decoded: serde_json::Value = serde_json::from_slice(&plaintext).unwrap(); + assert_eq!(decoded["env"], fixture["env"]); + } + } + + #[test] + fn sealed_env_round_trips_and_binds_info() { + let secret = StaticSecret::random_from_rng(rand::thread_rng()); + let public = PublicKey::from(&secret).to_bytes(); + let info = b"dstack-sealed-env-v1:3:0"; + let payload = sealed_env_encrypt(&public, info, b"secret env").unwrap(); + assert_eq!( + sealed_env_decrypt(&secret.to_bytes(), info, &payload).unwrap(), + b"secret env" + ); + assert!( + sealed_env_decrypt(&secret.to_bytes(), b"dstack-sealed-env-v1:4:0", &payload).is_err() + ); + let mut tampered = payload; + *tampered.last_mut().unwrap() ^= 1; + assert!(sealed_env_decrypt(&secret.to_bytes(), info, &tampered).is_err()); + } } diff --git a/dstack-util/src/host_api.rs b/dstack-util/src/host_api.rs index 6bcc270a6..ba85c5d41 100644 --- a/dstack-util/src/host_api.rs +++ b/dstack-util/src/host_api.rs @@ -9,13 +9,15 @@ use dstack_types::{ Platform, }; use host_api::{ - client::{new_client, DefaultClient}, + client::{new_client, DefaultClient, StatusError}, Notification, }; use ra_tls::attestation::validate_tcb; use sodiumbox::{generate_keypair, open_sealed_box, PUBLICKEYBYTES}; use tracing::warn; +const HOST_API_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + pub(crate) struct KeyProvision { pub sk: [u8; 32], pub mr: [u8; 32], @@ -78,6 +80,29 @@ impl HostApi { } } + /// Close the host's env update window for this boot, handing it the env + /// re-encrypted to the current key epochs, or an empty one when nothing + /// changed. An error means the window may still be open. + pub async fn update_sealed_env(&self, sealed_env: Vec) -> Result<()> { + if !matches!(Platform::detect_or_dstack(), Platform::Dstack) { + return Ok(()); + } + let Some(client) = &self.client else { + return Ok(()); + }; + let response = tokio::time::timeout( + HOST_API_TIMEOUT, + client.update_sealed_env(host_api::SealedEnv { sealed_env }), + ) + .await + .context("Timed out updating the sealed env on the host")?; + match response { + // A VMM predating UpdateSealedEnv has no window to close. + Err(err) if is_unknown_method(&err) => Ok(()), + response => response.map(drop), + } + } + pub async fn get_sealing_key(&self) -> Result { let (pk, sk) = generate_keypair(); let mut report_data = [0u8; 64]; @@ -122,3 +147,80 @@ impl HostApi { Ok(KeyProvision { sk, mr }) } } + +/// Whether the host does not implement the method at all: a VMM since 0.6 +/// answers 404, and older ones 400 with the pRPC dispatcher's fixed +/// "Service not found" message. +fn is_unknown_method(err: &anyhow::Error) -> bool { + err.chain() + .find_map(|cause| cause.downcast_ref::()) + .is_some_and(|err| { + err.status == 404 + || (err.status == 400 + && err + .message + .as_deref() + .is_some_and(|message| message.starts_with("Service not found: "))) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + + /// A host API answering one request with `status` and a JSON `body`. + fn host(status: u16, body: String) -> HostApi { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + let mut chunk = [0; 4096]; + while !request.windows(4).any(|w| w == b"\r\n\r\n") { + let read = stream.read(&mut chunk).unwrap(); + request.extend_from_slice(&chunk[..read]); + } + write!( + stream, + "HTTP/1.1 {status} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .unwrap(); + }); + HostApi::new(Some(url), None) + } + + #[tokio::test] + async fn only_an_answer_or_a_host_without_a_window_closes_the_env_window() { + assert!(matches!(Platform::detect_or_dstack(), Platform::Dstack)); + let unknown = r#"{"error":"Service not found: UpdateSealedEnv"}"#; + for (status, body, closed) in [ + (200, "", true), + // A VMM since 0.6, and one before, that does not know the method. + (404, unknown, true), + (400, unknown, true), + // Anything else may leave the window open. + (400, r#"{"error":"VM not found"}"#, false), + (500, "{}", false), + ] { + let result = host(status, body.into()) + .update_sealed_env(b"env".to_vec()) + .await; + assert_eq!(result.is_ok(), closed, "{status} {body}"); + } + // No host listening at all. + let gone = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", gone.local_addr().unwrap()); + drop(gone); + assert!(HostApi::new(Some(url), None) + .update_sealed_env(Vec::new()) + .await + .is_err()); + // No host API configured means no window. + HostApi::new(None, None) + .update_sealed_env(Vec::new()) + .await + .unwrap(); + } +} diff --git a/dstack-util/src/main.rs b/dstack-util/src/main.rs index fa839c379..cc1a4bfc4 100644 --- a/dstack-util/src/main.rs +++ b/dstack-util/src/main.rs @@ -431,6 +431,7 @@ fn make_app_keys( Ok(AppKeys { disk_crypt_key: sha256(&disk_key.serialize_der()).to_vec(), env_crypt_key: vec![], + sealed_env_key: vec![], k256_key: k256_key.to_bytes().to_vec(), k256_signature: vec![], gateway_app_id: "".to_string(), diff --git a/dstack-util/src/system_setup.rs b/dstack-util/src/system_setup.rs index 7b7d02986..7e9e11630 100644 --- a/dstack-util/src/system_setup.rs +++ b/dstack-util/src/system_setup.rs @@ -19,15 +19,12 @@ use dstack_kms_rpc as rpc; use dstack_types::{ shared_filenames::{ APP_COMPOSE, APP_KEYS, DECRYPTED_ENV, DECRYPTED_ENV_JSON, ENCRYPTED_ENV, - HOST_SHARED_DIR_NAME, HOST_SHARED_DISK_LABEL, INSTANCE_INFO, SYS_CONFIG, USER_CONFIG, + HOST_SHARED_DIR_NAME, HOST_SHARED_DISK_LABEL, INSTANCE_INFO, SEALED_ENV, SYS_CONFIG, + USER_CONFIG, }, KeyProvider, KeyProviderInfo, }; use fs_err as fs; -use luks2::{ - LuksAf, LuksConfig, LuksDigest, LuksHeader, LuksJson, LuksKdf, LuksKeyslot, LuksSegment, - LuksSegmentSize, -}; use ra_rpc::{ client::{CertInfo, RaClient, RaClientConfig}, Attestation, @@ -85,6 +82,12 @@ async fn sign_cert_request( } mod config_id_verifier; +mod luks; +use luks::{load_luks_header, sync_luks_epochs, write_luks_epochs, PAYLOAD_OFFSET}; +mod sealed_env; +#[cfg(test)] +use sealed_env::seal_env; +use sealed_env::{env_crypt_key, seal_env_at_epochs, sealed_env_info_at_epochs, SealedEnv}; #[derive(clap::Parser)] /// Prepare full disk encryption @@ -215,6 +218,10 @@ impl HostShareDir { self.base_dir.join(ENCRYPTED_ENV) } + fn sealed_env_file(&self) -> PathBuf { + self.base_dir.join(SEALED_ENV) + } + fn sys_config_file(&self) -> PathBuf { self.base_dir.join(SYS_CONFIG) } @@ -224,11 +231,34 @@ impl HostShareDir { } } +fn env_file_exists(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_file() => Ok(true), + Ok(_) => bail!("env {} is not a regular file", path.display()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => { + Err(error).with_context(|| format!("failed to inspect env {}", path.display())) + } + } +} + +fn read_optional_env(path: &Path) -> Result>> { + if !env_file_exists(path)? { + return Ok(None); + } + match fs::read(path) { + Ok(bytes) => Ok(Some(bytes)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error).with_context(|| format!("failed to read env {}", path.display())), + } +} + struct HostShared { dir: HostShareDir, sys_config: SysConfig, app_compose: AppCompose, encrypted_env: Vec, + sealed_env: Option>, instance_info: InstanceInfo, } @@ -279,12 +309,18 @@ impl HostShared { } else { InstanceInfo::default() }; - let encrypted_env = fs::read(host_shared_dir.encrypted_env_file()).unwrap_or_default(); + let sealed_env = read_optional_env(&host_shared_dir.sealed_env_file())?; + let encrypted_env = if sealed_env.is_some() { + Vec::new() + } else { + read_optional_env(&host_shared_dir.encrypted_env_file())?.unwrap_or_default() + }; Ok(Self { dir: host_shared_dir.clone(), sys_config, app_compose, encrypted_env, + sealed_env, instance_info, }) } @@ -361,7 +397,16 @@ impl HostShared { copy(APP_COMPOSE, SZ_1KB * 256, false)?; copy(SYS_CONFIG, SZ_1KB * 32, false)?; copy(INSTANCE_INFO, SZ_1KB * 10, true)?; - copy(ENCRYPTED_ENV, SZ_1KB * 256, true)?; + if env_file_exists(&host_shared_dir.join(SEALED_ENV))? { + copy(SEALED_ENV, SZ_1KB * 384, false)?; + } else { + match fs::remove_file(host_shared_copy_dir.join(SEALED_ENV)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("failed to remove stale sealed env copy"), + } + copy(ENCRYPTED_ENV, SZ_1KB * 256, true)?; + } copy(USER_CONFIG, SZ_1MB, true)?; cmd! { info "Unmounting host-shared"; @@ -369,6 +414,135 @@ impl HostShared { }?; HostShared::load(host_shared_copy_dir) } + + /// Request app keys from one KMS, of the given (root, app) epochs or the + /// current ones when `epochs` is `None`. + /// + /// Only the request for the current keys extends RTMR3 and takes the KMS + /// root CA from the KMS; a request for older epochs pins `trusted_ca`, the + /// CA this boot already established, and the CA of those epochs that the + /// KMS returns is not trusted for anything. + async fn request_app_keys_from_kms_url( + &self, + kms_url: String, + epochs: Option<(u32, u32)>, + trusted_ca: Option, + ) -> Result { + info!("Requesting app keys of epochs {epochs:?} from KMS: {kms_url}"); + let measure = trusted_ca.is_none(); + let tmp_ca = { + info!("Getting temp ca cert"); + let client = RaClient::new(kms_url.clone(), true)?; + let kms_client = dstack_kms_rpc::kms_client::KmsClient::new(client); + kms_client + .get_temp_ca_cert() + .await + .context("Failed to get temp ca cert")? + }; + let root_ca = trusted_ca.unwrap_or(tmp_ca.ca_cert); + let cert_pair = generate_ra_cert(tmp_ca.temp_ca_cert.clone(), tmp_ca.temp_ca_key.clone())?; + let ra_client = RaClientConfig::builder() + .tls_no_check(false) + .tls_built_in_root_certs(false) + .remote_uri(kms_url.clone()) + .tls_client_cert(cert_pair.cert_pem) + .tls_client_key(cert_pair.key_pem) + .tls_ca_cert(root_ca.clone()) + .maybe_pccs_url(self.sys_config.pccs_url.clone()) + .cert_validator(Box::new(move |cert| { + let Some(cert) = cert else { + bail!("Missing server cert"); + }; + let Some(usage) = cert.special_usage else { + bail!("Missing server cert usage"); + }; + if usage != "kms:rpc" { + bail!("Invalid server cert usage: {usage}"); + } + if let Some(att) = cert.attestation.as_ref().filter(|_| measure) { + let kms_info = att + .decode_app_info(false) + .context("Failed to decode app_info")?; + emit_runtime_event("mr-kms", &kms_info.mr_aggregated) + .context("Failed to extend mr-kms to RTMR3")?; + } + Ok(()) + })) + .build() + .into_client() + .context("Failed to create client")?; + let kms_client = dstack_kms_rpc::kms_client::KmsClient::new(ra_client); + let response = kms_client + .get_app_key(rpc::GetAppKeyRequest { + api_version: 1, + vm_config: self.sys_config.vm_config.clone(), + root_epoch: epochs.map(|(root, _)| root), + app_epoch: epochs.map(|(_, app)| app), + }) + .await + .context("Failed to get app key")?; + if let Some((root, app)) = epochs { + if (response.root_epoch, response.app_epoch) != (root, app) { + bail!( + "KMS returned epochs ({}, {}) instead of the requested ({root}, {app})", + response.root_epoch, + response.app_epoch + ); + } + } + + if measure { + emit_runtime_event("os-image-hash", &response.os_image_hash) + .context("Failed to extend os-image-hash to RTMR3")?; + } + + let (_, ca_pem) = x509_parser::pem::parse_x509_pem(root_ca.as_bytes()) + .context("Failed to parse ca cert")?; + let x509 = ca_pem.parse_x509().context("Failed to parse ca cert")?; + let root_pubkey = x509.public_key().raw.to_vec(); + + Ok(AppKeys { + ca_cert: root_ca, + disk_crypt_key: response.disk_crypt_key, + env_crypt_key: response.env_crypt_key, + sealed_env_key: response.sealed_env_key, + k256_key: response.k256_key, + k256_signature: response.k256_signature, + gateway_app_id: response.gateway_app_id, + key_provider: KeyProvider::Kms { + url: kms_url, + pubkey: root_pubkey, + tmp_ca_key: tmp_ca.temp_ca_key, + tmp_ca_cert: tmp_ca.temp_ca_cert, + root_epoch: response.root_epoch, + app_epoch: response.app_epoch, + }, + }) + } + + /// Request the app keys of older epochs from the KMS that served `keys`. + async fn request_historical_keys( + &self, + keys: &AppKeys, + root: u32, + app: u32, + ) -> Result { + let KeyProvider::Kms { url, .. } = &keys.key_provider else { + bail!("only KMS-provided keys have older epochs"); + }; + self.request_app_keys_from_kms_url( + url.clone(), + Some((root, app)), + Some(keys.ca_cert.clone()), + ) + .await + .with_context(|| { + format!( + "failed to get the keys of root epoch {root}, app epoch {app}; \ + the policy of those epochs must still allow this instance" + ) + }) + } } // The gateway's WireGuard fields are rendered into a wg-quick config, and @@ -869,82 +1043,6 @@ impl<'a> Stage0<'a> { self.shared.dir.join(APP_KEYS) } - async fn request_app_keys_from_kms_url(&self, kms_url: String) -> Result { - info!("Requesting app keys from KMS: {kms_url}"); - let tmp_ca = { - info!("Getting temp ca cert"); - let client = RaClient::new(kms_url.clone(), true)?; - let kms_client = dstack_kms_rpc::kms_client::KmsClient::new(client); - kms_client - .get_temp_ca_cert() - .await - .context("Failed to get temp ca cert")? - }; - let cert_pair = generate_ra_cert(tmp_ca.temp_ca_cert.clone(), tmp_ca.temp_ca_key.clone())?; - let ra_client = RaClientConfig::builder() - .tls_no_check(false) - .tls_built_in_root_certs(false) - .remote_uri(kms_url.clone()) - .tls_client_cert(cert_pair.cert_pem) - .tls_client_key(cert_pair.key_pem) - .tls_ca_cert(tmp_ca.ca_cert.clone()) - .maybe_pccs_url(self.shared.sys_config.pccs_url.clone()) - .cert_validator(Box::new(|cert| { - let Some(cert) = cert else { - bail!("Missing server cert"); - }; - let Some(usage) = cert.special_usage else { - bail!("Missing server cert usage"); - }; - if usage != "kms:rpc" { - bail!("Invalid server cert usage: {usage}"); - } - if let Some(att) = &cert.attestation { - let kms_info = att - .decode_app_info(false) - .context("Failed to decode app_info")?; - emit_runtime_event("mr-kms", &kms_info.mr_aggregated) - .context("Failed to extend mr-kms to RTMR3")?; - } - Ok(()) - })) - .build() - .into_client() - .context("Failed to create client")?; - let kms_client = dstack_kms_rpc::kms_client::KmsClient::new(ra_client); - let response = kms_client - .get_app_key(rpc::GetAppKeyRequest { - api_version: 1, - vm_config: self.shared.sys_config.vm_config.clone(), - }) - .await - .context("Failed to get app key")?; - - emit_runtime_event("os-image-hash", &response.os_image_hash) - .context("Failed to extend os-image-hash to RTMR3")?; - - let (_, ca_pem) = x509_parser::pem::parse_x509_pem(tmp_ca.ca_cert.as_bytes()) - .context("Failed to parse ca cert")?; - let x509 = ca_pem.parse_x509().context("Failed to parse ca cert")?; - let root_pubkey = x509.public_key().raw.to_vec(); - - let keys = AppKeys { - ca_cert: tmp_ca.ca_cert, - disk_crypt_key: response.disk_crypt_key, - env_crypt_key: response.env_crypt_key, - k256_key: response.k256_key, - k256_signature: response.k256_signature, - gateway_app_id: response.gateway_app_id, - key_provider: KeyProvider::Kms { - url: kms_url, - pubkey: root_pubkey, - tmp_ca_key: tmp_ca.temp_ca_key, - tmp_ca_cert: tmp_ca.temp_ca_cert, - }, - }; - Ok(keys) - } - async fn request_app_keys_from_kms(&self) -> Result { if self.shared.sys_config.kms_urls.is_empty() { bail!("No KMS URLs are set"); @@ -953,7 +1051,10 @@ impl<'a> Stage0<'a> { let mut error = anyhow!("unknown error"); for (i, kms_url) in self.shared.sys_config.kms_urls.iter().enumerate() { let kms_url = format!("{kms_url}/prpc"); - let response = self.request_app_keys_from_kms_url(kms_url.clone()).await; + let response = self + .shared + .request_app_keys_from_kms_url(kms_url.clone(), None, None) + .await; match response { Ok(response) => { break 'out response; @@ -1159,7 +1260,7 @@ impl<'a> Stage0<'a> { initialized } - async fn mount_data_disk(&self, disk_crypt_key: &str, opts: &DstackOptions) -> Result<()> { + async fn mount_data_disk(&self, keys: &AppKeys, opts: &DstackOptions) -> Result<()> { let name = "dstack_data_disk"; let mount_point = &self.args.mount_point; @@ -1181,7 +1282,7 @@ impl<'a> Stage0<'a> { if opts.storage_encrypted { info!("Setting up disk encryption"); - self.luks_setup(disk_crypt_key, name)?; + self.luks_setup(keys, name).await?; } else { info!("Skipping disk encryption as requested by kernel cmdline"); } @@ -1211,7 +1312,7 @@ impl<'a> Stage0<'a> { if opts.storage_encrypted { info!("Mounting encrypted data disk"); - self.open_encrypted_volume(disk_crypt_key, name)?; + self.open_encrypted_volume(keys, name).await?; } else { info!("Mounting unencrypted data disk"); } @@ -1275,8 +1376,9 @@ impl<'a> Stage0<'a> { Ok(()) } - fn luks_setup(&self, disk_crypt_key: &str, name: &str) -> Result<()> { + async fn luks_setup(&self, keys: &AppKeys, name: &str) -> Result<()> { let root_hd = &self.args.device; + let disk_crypt_key = hex::encode(&keys.disk_crypt_key); let sector_offset = PAYLOAD_OFFSET / 512; cmd! { info "Formatting encrypted disk"; @@ -1291,12 +1393,17 @@ impl<'a> Stage0<'a> { $name; } .or(Err(anyhow!("Failed to setup luks volume")))?; - self.open_encrypted_volume(disk_crypt_key, name) + // An epoch-0 disk carries no token, so older OS images still accept it. + if let Some(root) = kms_root_epoch(keys).filter(|root| *root > 0 || keys.app_epoch() > 0) { + write_luks_epochs(root_hd, 0, root, keys.app_epoch())?; + } + self.open_encrypted_volume(keys, name).await } - fn open_encrypted_volume(&self, disk_crypt_key: &str, name: &str) -> Result<()> { + /// Open the data disk after bringing it to the current key epochs. + async fn open_encrypted_volume(&self, keys: &AppKeys, name: &str) -> Result<()> { let root_hd = &self.args.device; - let disk_crypt_key = disk_crypt_key.trim(); + let disk_crypt_key = hex::encode(&keys.disk_crypt_key); // Create a private tmpfs mount to ensure the header stays in-memory. let tmp_hdr_dir = "/tmp/dstack-luks-header"; let in_mem_hdr = format!("{tmp_hdr_dir}/luks-header"); @@ -1313,13 +1420,21 @@ impl<'a> Stage0<'a> { info "Mounting tmpfs for in-memory LUKS header"; mkdir -p $tmp_hdr_dir; mount -t tmpfs -o size=64M,mode=0700,nosuid,nodev,noexec tmpfs $tmp_hdr_dir; - info "Loading the LUKS2 header"; - cryptsetup luksHeaderBackup --header-backup-file=$in_mem_hdr $root_hd; } - .context("Failed to load LUKS2 header")?; + .context("Failed to mount tmpfs for the LUKS2 header")?; - let hdr_file = fs::File::open(&in_mem_hdr).context("Failed to open LUKS2 header")?; - validate_luks2_headers(hdr_file).context("Failed to validate LUKS2 header")?; + let hdr = Path::new(&in_mem_hdr); + let header = load_luks_header(root_hd, hdr)?; + if let Some(current) = kms_root_epoch(keys) { + let old_key = async |root, app| { + let old = self.shared.request_historical_keys(keys, root, app).await?; + Ok(hex::encode(&old.disk_crypt_key)) + }; + let target = (current, keys.app_epoch()); + sync_luks_epochs(root_hd, hdr, header, &disk_crypt_key, target, old_key) + .await + .context("failed to migrate the data disk key")?; + } cmd! { info "Opening the device"; @@ -1457,8 +1572,7 @@ impl<'a> Stage0<'a> { opts.storage_encrypted, opts.storage_fs ); - self.mount_data_disk(&hex::encode(&app_keys.disk_crypt_key), &opts) - .await?; + self.mount_data_disk(&app_keys, &opts).await?; self.setup_swap(self.shared.app_compose.swap_size, &opts) .await?; self.vmm @@ -1483,25 +1597,80 @@ impl<'a> Stage0<'a> { } impl Stage1<'_> { - fn decrypt_env_vars( - &self, - key: &[u8], - ciphertext: &[u8], - allowed: &BTreeSet, - ) -> Result> { - let vars = if !ciphertext.is_empty() { - info!("Processing encrypted env"); - let env_crypt_key: [u8; 32] = key.try_into().ok().context( - "encrypted env is present but the key provider gave no valid env crypt key", + /// Decrypt the host-supplied env, preferring the sealed format. + /// + /// Also returns the env re-sealed to the current epochs when it was + /// encrypted to older ones, or an empty one when it needs no change. + async fn decrypt_env(&self) -> Result<(Vec, Vec)> { + let current_secret = self.keys.sealed_env_key.as_slice(); + if let Some(sealed_env) = &self.shared.sealed_env { + let envelope: SealedEnv = + serde_json::from_slice(sealed_env).context("failed to parse sealed env")?; + if envelope.version != 1 { + bail!("unsupported sealed env version: {}", envelope.version); + } + if envelope.enc.len() != 32 { + bail!("sealed env encapsulated key must be 32 bytes"); + } + let payload = [envelope.enc, envelope.ciphertext].concat(); + let (current_root, current_app) = (self.keys.root_epoch(), self.keys.app_epoch()); + if envelope.root_epoch > current_root || envelope.app_epoch > current_app { + bail!( + "sealed env epoch (root {}, app {}) exceeds current epoch (root {current_root}, app {current_app})", + envelope.root_epoch, + envelope.app_epoch + ); + } + let is_current = + (envelope.root_epoch, envelope.app_epoch) == (current_root, current_app); + let historical; + let secret = if is_current { + current_secret + } else { + historical = self + .shared + .request_historical_keys(&self.keys, envelope.root_epoch, envelope.app_epoch) + .await?; + historical.sealed_env_key.as_slice() + }; + let plaintext = crate::crypto::sealed_env_decrypt( + secret, + &sealed_env_info_at_epochs(envelope.root_epoch, envelope.app_epoch), + &payload, )?; - let decrypted_json = - dh_decrypt(env_crypt_key, ciphertext).context("Failed to decrypt env file")?; - crate::parse_env_file::parse_env(&decrypted_json, allowed)? - } else { - info!("No encrypted env, using default"); - Default::default() + if is_current { + return Ok((plaintext, Vec::new())); + } + let sealed = seal_env_at_epochs(current_secret, current_root, current_app, &plaintext)?; + return Ok((plaintext, sealed)); + } + let ciphertext = &self.shared.encrypted_env; + let current_key = env_crypt_key(&self.keys.env_crypt_key)?; + let plaintext = match dh_decrypt(current_key, ciphertext) { + Ok(plaintext) => return Ok((plaintext, Vec::new())), + Err(_) => { + let current = self.keys.root_epoch(); + // Legacy encryption is only issued at app epoch zero. + let epochs = (0..=current) + .rev() + .filter(|root| *root != current || self.keys.app_epoch() != 0); + decrypt_historical_env(ciphertext, epochs, async |root| { + Ok(self + .shared + .request_historical_keys(&self.keys, root, 0) + .await? + .env_crypt_key) + }) + .await? + } }; - Ok(vars) + let sealed = seal_env_at_epochs( + current_secret, + self.keys.root_epoch(), + self.keys.app_epoch(), + &plaintext, + )?; + Ok((plaintext, sealed)) } fn write_env_file(&self, env_vars: &BTreeMap) -> Result<()> { @@ -1518,7 +1687,7 @@ impl Stage1<'_> { Ok(()) } - fn unseal_env_vars(&self) -> Result> { + async fn unseal_env_vars(&self) -> Result> { let allowed_envs: BTreeSet = self .shared .app_compose @@ -1526,18 +1695,32 @@ impl Stage1<'_> { .iter() .cloned() .collect(); - // Decrypt env file - let decrypted_env = self.decrypt_env_vars( - &self.keys.env_crypt_key, - &self.shared.encrypted_env, - &allowed_envs, - )?; + let (decrypted_env, sealed) = + if self.shared.sealed_env.is_none() && self.shared.encrypted_env.is_empty() { + info!("No encrypted env, using default"); + (Default::default(), Vec::new()) + } else { + info!("Processing encrypted env"); + let (plaintext, sealed) = self.decrypt_env().await?; + ( + crate::parse_env_file::parse_env(&plaintext, &allowed_envs)?, + sealed, + ) + }; self.write_env_file(&decrypted_env)?; + // Sent on every boot, empty when nothing changed: it closes the host's + // env update window, which must be shut before any application code + // runs, or that code could replace the env the next boot reads. + close_env_window( + || self.vmm.update_sealed_env(sealed.clone()), + &ENV_WINDOW_RETRY_DELAYS, + ) + .await?; Ok(decrypted_env) } async fn setup(&self) -> Result<()> { - let _envs = self.unseal_env_vars()?; + let _envs = self.unseal_env_vars().await?; self.link_files()?; self.setup_socket_dir()?; self.setup_guest_agent_config()?; @@ -1617,299 +1800,60 @@ impl Stage1<'_> { } } -macro_rules! const_pad { - ($s:expr, $len:expr) => { - const { - assert!($s.len() <= $len, "The s is too long"); - let mut padded: [u8; $len] = [0; $len]; - let mut i = 0; - while i < $s.len() { - padded[i] = $s[i]; - i += 1; - } - padded - } - }; -} - -const PAYLOAD_OFFSET: u64 = 16777216; - -fn validate_luks2_headers(mut reader: impl std::io::Read) -> Result<()> { - validate_single_luks2_header(&mut reader, 0)?; - validate_single_luks2_header(&mut reader, 1)?; - Ok(()) +/// Waits between attempts to close the env update window. +const ENV_WINDOW_RETRY_DELAYS: [Duration; 4] = [ + Duration::from_secs(1), + Duration::from_secs(2), + Duration::from_secs(4), + Duration::from_secs(8), +]; + +/// Close the host's env update window, retrying `update` after each delay +/// while the host gives no answer. Fails if the window may still be open. +async fn close_env_window(mut update: F, delays: &[Duration]) -> Result<()> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + for delay in delays { + match update().await { + Ok(()) => return Ok(()), + Err(err) => warn!("failed to close the env update window, retrying: {err:#}"), + } + tokio::time::sleep(*delay).await; + } + update().await.context( + "the host did not confirm that the env update window is closed; refusing to start applications", + ) } -fn validate_single_luks2_header(mut reader: impl std::io::Read, hdr_ind: u64) -> Result<()> { - let mut hdr_data = vec![0u8; 4096]; - reader - .read_exact(&mut hdr_data) - .context("Failed to read LUKS header")?; - let header = - LuksHeader::read_from(&mut &hdr_data[..]).context("Failed to decode LUKS header")?; - let LuksHeader { - magic, - version, - hdr_size, - seqid: _, - label, - csum_alg, - salt: _, - uuid: _, - subsystem, - hdr_offset, - csum: _, - .. - } = header; - - let expected_magic = match hdr_ind { - 0 => [76, 85, 75, 83, 186, 190], - 1 => [83, 75, 85, 76, 186, 190], - _ => bail!("Invalid LUKS header index: {hdr_ind}"), - }; - if magic != expected_magic { - bail!("Invalid LUKS magic: {magic:?}"); - } - if version != 2 { - bail!("Invalid LUKS version: {version}"); - } - if label != [0; 48] { - bail!("Invalid LUKS label: {:?}", label); - } - if csum_alg != const_pad!(b"sha256", 32) { - bail!("Invalid LUKS checksum algorithm"); - } - if subsystem != [0; 48] { - bail!("Invalid LUKS subsystem"); - } - if hdr_offset != hdr_ind * hdr_size { - bail!("Invalid LUKS header offset: {hdr_offset}"); - } - if !(4096..=1024 * 1024 * 16).contains(&hdr_size) { - bail!("Invalid LUKS header size: {hdr_size}"); - } - - // Check JSON - let json_size = hdr_size - 4096; - let mut jsn_data = vec![0u8; json_size as usize]; - reader - .read_exact(&mut jsn_data) - .context("Failed to read LUKS JSON")?; - let json_end = jsn_data - .iter() - .position(|&b| b == 0) - .unwrap_or(jsn_data.len()); - jsn_data.truncate(json_end); - - let json = LuksJson::read_from(&mut &jsn_data[..]).context("Failed to decode LUKS JSON")?; - let LuksJson { - keyslots, - tokens, - segments, - digests, - config: - LuksConfig { - json_size: _, - keyslots_size: _, - flags, - requirements, - }, - } = json; - - if keyslots.len() != 1 { - bail!("Invalid LUKS keyslots"); - } - if !tokens.is_empty() { - bail!("Invalid LUKS tokens"); - } - if segments.len() != 1 { - bail!("Invalid LUKS segments"); - } - if digests.len() != 1 { - bail!("Invalid LUKS digests"); - } - if flags.is_some() { - bail!("Invalid LUKS flags"); - } - if requirements.is_some() { - bail!("Invalid LUKS requirements"); - } - - { - let first_keyslot = keyslots.get(&0).context("no LUKS keyslot")?; - let LuksKeyslot::luks2 { - key_size, - area, - kdf, - af, - priority, - } = first_keyslot; - if area.encryption() != "aes-xts-plain64" { - bail!("Invalid LUKS keyslot encryption: {}", area.encryption()); - } - // Pin where the encrypted key material is read from. The binary area - // must sit between the two header copies and the encrypted payload; - // otherwise a host with raw disk access could redirect it elsewhere. - let area_end = area.offset().checked_add(area.size()); - if area.offset() < 2 * hdr_size || area_end.is_none_or(|end| end > PAYLOAD_OFFSET) { - bail!( - "Invalid LUKS keyslot area: offset={} size={}", - area.offset(), - area.size() - ); - } - if *key_size != 64 { - bail!("Invalid LUKS keyslot key size: {key_size}"); - } - if area.key_size() != 64 { - bail!("Invalid LUKS keyslot key size: {}", area.key_size()); - } - { - let LuksKdf::pbkdf2 { - hash, - iterations: _, - salt: _, - } = kdf - else { - bail!("Invalid LUKS keyslot KDF"); - }; - if hash != "sha256" { - bail!("Invalid LUKS keyslot hash: {hash}"); - } - } - { - let LuksAf::luks1 { hash, stripes } = af; - if hash != "sha256" { - bail!("Invalid LUKS keyslot hash: {hash}"); - } - if *stripes != 4000 { - bail!("Invalid LUKS keyslot stripes: {stripes}"); - } - } - if priority.is_some() { - bail!("Invalid LUKS keyslot priority"); - } - } - - { - let first_segment = segments.get(&0).context("no LUKS segment")?; - let LuksSegment::crypt { - offset, - size, - iv_tweak, - encryption, - sector_size, - integrity, - flags, - } = first_segment; - if *offset != PAYLOAD_OFFSET { - bail!("Invalid LUKS segment offset"); - } - if *size != LuksSegmentSize::dynamic { - bail!("Invalid LUKS segment size"); - } - if *iv_tweak != 0 { - bail!("Invalid LUKS segment IV tweak"); - } - if encryption != "aes-xts-plain64" { - bail!("Invalid LUKS segment encryption"); - } - if *sector_size != 512 { - bail!("Invalid LUKS segment sector size"); - } - if integrity.is_some() { - bail!("Invalid LUKS segment integrity"); - } - if flags.is_some() { - bail!("Invalid LUKS segment flags"); - } - } - { - let first_digest = digests.get(&0).context("no LUKS digest")?; - let LuksDigest::pbkdf2 { - keyslots, - segments, - hash, - digest: _, - iterations: _, - salt: _, - } = first_digest; - if hash != "sha256" { - bail!("Invalid LUKS digest hash: {hash}"); - } - if keyslots != &[0] { - bail!("Invalid LUKS digest keyslots: {keyslots:?}"); - } - if segments != &[0] { - bail!("Invalid LUKS digest segments: {segments:?}"); +async fn decrypt_historical_env( + ciphertext: &[u8], + epochs: I, + mut key_at: F, +) -> Result> +where + I: IntoIterator, + F: FnMut(u32) -> Fut, + Fut: std::future::Future>>, +{ + for epoch in epochs { + let decrypted = key_at(epoch) + .await + .and_then(|key| dh_decrypt(env_crypt_key(&key)?, ciphertext)); + match decrypted { + Ok(plaintext) => return Ok(plaintext), + Err(err) => warn!("failed to open legacy env at root epoch {epoch}: {err:#}"), } } - Ok(()) + bail!("failed to decrypt legacy env with accessible historical keys") } -#[test] -fn test_validate_luks2_header() { - let header_data = include_bytes!("../tests/fixtures/luks_header_good").to_vec(); - validate_luks2_headers(&mut &header_data[..]).expect("Failed to validate LUKS2 header"); - let header_data = include_bytes!("../tests/fixtures/luks_header_cipher_null").to_vec(); - let error = validate_luks2_headers(&mut &header_data[..]).unwrap_err(); - assert!(error - .to_string() - .contains("Invalid LUKS keyslot encryption")); -} - -#[test] -fn test_validate_luks2_header_rejects_out_of_range_keyslot_area() { - // Redirect the keyslot binary area below the header region. Same length - // so the surrounding header stays intact; "00768" parses to 768, which is - // inside the header copies (< 2 * hdr_size) rather than the metadata gap. - let mut header = include_bytes!("../tests/fixtures/luks_header_good").to_vec(); - let patched = patch_luks_json(&mut header, br#""offset":"32768""#, br#""offset":"00768""#); - assert_eq!(patched, 2, "expected to patch both header copies"); - let error = validate_luks2_headers(&mut &header[..]).unwrap_err(); - assert!(error.to_string().contains("Invalid LUKS keyslot area")); -} - -/// Replace `needle` with `replacement` inside every LUKS JSON region, keeping -/// each region the same length by giving back trailing NUL padding. -#[cfg(test)] -fn patch_luks_json(header: &mut [u8], needle: &[u8], replacement: &[u8]) -> usize { - assert!(replacement.len() >= needle.len()); - let grow = replacement.len() - needle.len(); - let mut patched = 0; - let mut i = 0; - while i + needle.len() <= header.len() { - if &header[i..i + needle.len()] != needle { - i += 1; - continue; - } - let tail = &mut header[i..]; - let end = tail.iter().position(|b| *b == 0).expect("NUL padding"); - assert!(end + grow < tail.len(), "not enough NUL padding"); - tail.copy_within(needle.len()..end, replacement.len()); - tail[..replacement.len()].copy_from_slice(replacement); - patched += 1; - i += replacement.len(); - } - patched -} - -#[test] -fn test_validate_luks2_header_rejects_keyslot_area_that_overflows() { - let mut header = include_bytes!("../tests/fixtures/luks_header_good").to_vec(); - // 2**64 - 32768 + 1000: added to the accepted offset 32768 it wraps to - // 1000, which is below PAYLOAD_OFFSET. - let patched = patch_luks_json( - &mut header, - br#""size":"258048""#, - br#""size":"18446744073709519848""#, - ); - assert_eq!(patched, 2, "expected to patch both header copies"); - let error = validate_luks2_headers(&mut &header[..]).unwrap_err(); - assert!( - error.to_string().contains("Invalid LUKS keyslot area"), - "{error:#}" - ); +fn kms_root_epoch(keys: &AppKeys) -> Option { + match keys.key_provider { + KeyProvider::Kms { root_epoch, .. } => Some(root_epoch), + _ => None, + } } #[test] @@ -1940,3 +1884,235 @@ fn wireguard_fields_reject_config_syntax() { assert!(parse_wg_ip("10.0.0.2").is_ok()); assert!(parse_wg_ip("10.0.0.2/32\n[Interface]").is_err()); } + +#[cfg(test)] +mod sealed_env_file_tests { + use super::*; + + fn stage(args: &SetupArgs, sealed_env: Option>, encrypted_env: Vec) -> Stage1<'_> { + Stage1 { + args, + vmm: HostApi::default(), + shared: HostShared { + dir: HostShareDir::new(&args.work_dir), + sys_config: serde_json::from_value(serde_json::json!({"vm_config": ""})).unwrap(), + app_compose: serde_json::from_value(serde_json::json!({ + "manifest_version": 2, "name": "sealed-test", "runner": "docker-compose" + })) + .unwrap(), + instance_info: InstanceInfo::default(), + sealed_env, + encrypted_env, + }, + keys: AppKeys { + disk_crypt_key: vec![], + env_crypt_key: vec![9; 32], + sealed_env_key: vec![7; 32], + k256_key: vec![], + k256_signature: vec![], + gateway_app_id: String::new(), + ca_cert: String::new(), + key_provider: KeyProvider::Kms { + url: "http://127.0.0.1:1".into(), + pubkey: vec![], + tmp_ca_key: String::new(), + tmp_ca_cert: String::new(), + root_epoch: 3, + app_epoch: 0, + }, + }, + } + } + + #[tokio::test] + async fn guest_preserves_env_at_the_current_epoch() { + let dir = tempfile::tempdir().unwrap(); + let args = SetupArgs { + work_dir: dir.path().into(), + device: "unused".into(), + mount_point: "unused".into(), + }; + let plaintext = br#"{"env":[{"key":"MESSAGE","value":"test"}]}"#; + let sealed = seal_env(&[7; 32], 3, plaintext).unwrap(); + let current = stage(&args, Some(sealed), b"invalid legacy".to_vec()); + let (opened, update) = current.decrypt_env().await.unwrap(); + assert_eq!(opened, plaintext); + assert!(update.is_empty()); + + let public = x25519_dalek::PublicKey::from(&x25519_dalek::StaticSecret::from([9; 32])); + let legacy = crate::crypto::dh_encrypt(public.to_bytes(), plaintext).unwrap(); + let old = stage(&args, None, legacy.clone()); + let (opened, update) = old.decrypt_env().await.unwrap(); + assert_eq!(opened, plaintext); + assert!(update.is_empty()); + let mut without_sealed_key = stage(&args, None, legacy.clone()); + without_sealed_key.keys.sealed_env_key.clear(); + let (opened, update) = without_sealed_key.decrypt_env().await.unwrap(); + assert_eq!(opened, plaintext); + assert!(update.is_empty()); + + for invalid in [ + Vec::new(), + b"invalid JSON".to_vec(), + br#"{"version":2,"root_epoch":3,"enc":"","ciphertext":""}"#.to_vec(), + ] { + assert!(stage(&args, Some(invalid), legacy.clone()) + .decrypt_env() + .await + .is_err()); + } + let future = seal_env(&[7; 32], 4, plaintext).unwrap(); + assert!(stage(&args, Some(future), legacy) + .decrypt_env() + .await + .unwrap_err() + .to_string() + .contains("exceeds current")); + } + + /// Run `close_env_window` against `answers`, returning its result and how + /// often the host was asked. + async fn close_with(answers: Vec>) -> (Result<()>, usize) { + let mut answers = answers.into_iter(); + let mut calls = 0; + let result = close_env_window( + || { + calls += 1; + std::future::ready(answers.next().unwrap()) + }, + &[Duration::ZERO; 2], + ) + .await; + (result, calls) + } + + #[tokio::test] + async fn applications_start_only_once_the_env_window_is_closed() { + let (result, calls) = close_with(vec![Ok(())]).await; + result.unwrap(); + assert_eq!(calls, 1); + // A lost answer is retried. + let (result, calls) = close_with(vec![Err(anyhow!("connection reset")), Ok(())]).await; + result.unwrap(); + assert_eq!(calls, 2); + // No answer within the retries fails the boot. + let (result, calls) = close_with(vec![ + Err(anyhow!("timed out")), + Err(anyhow!("timed out")), + Err(anyhow!("connection refused")), + ]) + .await; + let err = format!("{:#}", result.unwrap_err()); + assert!(err.contains("refusing to start applications"), "{err}"); + assert!(err.contains("connection refused"), "{err}"); + assert_eq!(calls, 3); + } + + #[tokio::test] + async fn legacy_env_migration_skips_denied_intermediate_epochs() { + let secret = x25519_dalek::StaticSecret::from([8; 32]); + let public = x25519_dalek::PublicKey::from(&secret); + let plaintext = br#"{"env":[{"key":"SECRET","value":"synthetic"}]}"#; + let encrypted = crate::crypto::dh_encrypt(public.to_bytes(), plaintext).unwrap(); + let mut requested = Vec::new(); + let decrypted = decrypt_historical_env(&encrypted, (0..2).rev(), |epoch| { + requested.push(epoch); + std::future::ready(if epoch == 0 { + Ok(secret.to_bytes().to_vec()) + } else { + Err(anyhow::anyhow!("authorization denied")) + }) + }) + .await + .unwrap(); + assert_eq!(requested, [1, 0]); + assert_eq!(decrypted, plaintext); + assert!( + decrypt_historical_env(&encrypted, (0..2).rev(), |_| async { bail!("denied") }) + .await + .is_err() + ); + assert!( + decrypt_historical_env(&encrypted, (0..2).rev(), |_| async { Ok(vec![3; 32]) }) + .await + .is_err() + ); + } + + #[tokio::test] + async fn sealed_env_at_the_current_epochs_opens_and_binds_both_epochs() { + let dir = tempfile::tempdir().unwrap(); + let args = SetupArgs { + work_dir: dir.path().into(), + device: "unused".into(), + mount_point: "unused".into(), + }; + let plaintext = br#"{"env":[{"key":"VALUE","value":"epoch-test"}]}"#; + let sealed = seal_env_at_epochs(&[7; 32], 3, 2, plaintext).unwrap(); + let mut current = stage(&args, Some(sealed.clone()), vec![]); + if let KeyProvider::Kms { app_epoch, .. } = &mut current.keys.key_provider { + *app_epoch = 2; + } + let (opened, update) = current.decrypt_env().await.unwrap(); + assert_eq!(opened, plaintext); + assert!(update.is_empty()); + let mut tampered: Value = serde_json::from_slice(&sealed).unwrap(); + tampered["version"] = 2.into(); + current.shared.sealed_env = Some(serde_json::to_vec(&tampered).unwrap()); + let err = current.decrypt_env().await.unwrap_err(); + assert!( + err.to_string().contains("unsupported sealed env version"), + "{err}" + ); + let envelope: SealedEnv = serde_json::from_slice(&sealed).unwrap(); + assert_eq!(envelope.version, 1); + let payload = [envelope.enc.clone(), envelope.ciphertext.clone()].concat(); + assert!(crate::crypto::sealed_env_decrypt( + &[7; 32], + &sealed_env_info_at_epochs(3, 1), + &payload + ) + .is_err()); + assert!(crate::crypto::sealed_env_decrypt( + &[7; 32], + &sealed_env_info_at_epochs(2, 2), + &payload + ) + .is_err()); + } + + #[test] + fn sealed_file_selection_ignores_legacy_but_rejects_links() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join(SYS_CONFIG), br#"{"vm_config":""}"#).unwrap(); + fs::write( + dir.path().join(APP_COMPOSE), + br#"{"manifest_version":2,"name":"test","runner":"docker-compose"}"#, + ) + .unwrap(); + let sealed_path = dir.path().join(SEALED_ENV); + fs::write(&sealed_path, b"sealed bytes").unwrap(); + fs::create_dir(dir.path().join(ENCRYPTED_ENV)).unwrap(); + let shared = HostShared::load(dir.path()).unwrap(); + assert_eq!(shared.sealed_env.unwrap(), b"sealed bytes"); + assert!(shared.encrypted_env.is_empty()); + fs::remove_file(&sealed_path).unwrap(); + std::os::unix::fs::symlink("missing-target", &sealed_path).unwrap(); + assert!(env_file_exists(&sealed_path).is_err()); + assert!(HostShared::load(dir.path()).is_err()); + } + + #[test] + fn missing_empty_and_unreadable_env_are_distinct() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(SEALED_ENV); + assert_eq!(read_optional_env(&path).unwrap(), None); + fs::write(&path, []).unwrap(); + assert_eq!(read_optional_env(&path).unwrap(), Some(Vec::new())); + assert!( + serde_json::from_slice::(&read_optional_env(&path).unwrap().unwrap()) + .is_err() + ); + assert!(read_optional_env(dir.path()).is_err()); + } +} diff --git a/dstack-util/src/system_setup/luks.rs b/dstack-util/src/system_setup/luks.rs new file mode 100644 index 000000000..fa2567d63 --- /dev/null +++ b/dstack-util/src/system_setup/luks.rs @@ -0,0 +1,969 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! LUKS2 header validation and key epoch migration. +//! Fields validation does not inspect are not modeled; serde ignores them. +//! See . + +use std::{ + collections::HashMap, + io::Write, + path::Path, + process::{Command, Stdio}, +}; + +use anyhow::{bail, Context, Result}; +use fs_err as fs; +use safe_write::safe_write_with_mode; +use scopeguard::defer; +use serde::{de::IgnoredAny, Deserialize}; +use serde_with::{serde_as, DisplayFromStr}; +use tracing::info; + +/// Size of the binary header that precedes each JSON area. +const BINARY_HEADER_SIZE: u64 = 4096; + +struct LuksHeader { + pub magic: [u8; 6], + pub version: u16, + pub hdr_size: u64, + pub label: [u8; 48], + pub csum_alg: [u8; 32], + pub subsystem: [u8; 48], + pub hdr_offset: u64, +} + +impl LuksHeader { + /// Decodes the big-endian binary header. + pub fn parse(data: &[u8]) -> Result { + fn field(data: &[u8], offset: usize) -> Result<[u8; N]> { + let bytes = data + .get(offset..offset + N) + .context("LUKS header is truncated")?; + Ok(bytes.try_into()?) + } + Ok(Self { + magic: field(data, 0)?, + version: u16::from_be_bytes(field(data, 6)?), + hdr_size: u64::from_be_bytes(field(data, 8)?), + label: field(data, 24)?, + csum_alg: field(data, 72)?, + subsystem: field(data, 208)?, + hdr_offset: u64::from_be_bytes(field(data, 256)?), + }) + } +} + +#[derive(Deserialize)] +struct LuksJson { + pub keyslots: HashMap, + pub tokens: HashMap, + pub segments: HashMap, + pub digests: HashMap, + pub config: LuksConfig, +} + +/// Type of the only token dstack writes: the KMS key epochs of one keyslot. +const ROOT_EPOCH_TOKEN_TYPE: &str = "dstack-kms"; + +#[derive(Deserialize)] +struct LuksToken { + #[serde(rename = "type")] + pub kind: String, + pub keyslots: Vec, + #[serde(default)] + pub root_epoch: Option, + #[serde(default)] + pub app_epoch: u32, +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum LuksKeyslot { + Luks2 { + key_size: u16, + area: LuksArea, + kdf: LuksKdf, + af: LuksAf, + #[serde(default)] + priority: Option, + }, +} + +#[serde_as] +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum LuksArea { + Raw { + encryption: String, + key_size: u32, + #[serde_as(as = "DisplayFromStr")] + offset: u64, + #[serde_as(as = "DisplayFromStr")] + size: u64, + }, +} + +/// Salt and iterations are left unchecked on purpose: the passphrase is +/// high-entropy and KMS-derived, so attacker-chosen values buy nothing without +/// it (see security report #552). +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum LuksKdf { + Pbkdf2 { + hash: String, + }, + #[serde(other)] + Unsupported, +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum LuksAf { + Luks1 { hash: String, stripes: u16 }, +} + +#[serde_as] +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum LuksSegment { + Crypt { + #[serde_as(as = "DisplayFromStr")] + offset: u64, + size: String, + #[serde_as(as = "DisplayFromStr")] + iv_tweak: u64, + encryption: String, + sector_size: u16, + #[serde(default)] + integrity: Option, + #[serde(default)] + flags: Option, + }, +} + +#[serde_as] +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum LuksDigest { + Pbkdf2 { + #[serde_as(as = "Vec")] + keyslots: Vec, + #[serde_as(as = "Vec")] + segments: Vec, + hash: String, + }, +} + +#[serde_as] +#[derive(Deserialize)] +struct LuksConfig { + #[serde_as(as = "DisplayFromStr")] + pub keyslots_size: u64, + #[serde(default)] + pub flags: Option, + #[serde(default)] + pub requirements: Option, +} + +macro_rules! const_pad { + ($s:expr, $len:expr) => { + const { + assert!($s.len() <= $len, "The s is too long"); + let mut padded: [u8; $len] = [0; $len]; + let mut i = 0; + while i < $s.len() { + padded[i] = $s[i]; + i += 1; + } + padded + } + }; +} + +pub(super) const PAYLOAD_OFFSET: u64 = 16777216; + +fn cryptsetup(args: &[&str], device: &Path) -> Command { + let mut command = Command::new("cryptsetup"); + command.args(args).arg(device); + command +} + +/// Run `command` with `input` on stdin and return its exit code. +fn run_with_input(mut command: Command, input: &[u8]) -> Result> { + let mut child = command + .stdin(Stdio::piped()) + .spawn() + .context("failed to start cryptsetup")?; + child + .stdin + .take() + .context("cryptsetup stdin is unavailable")? + .write_all(input) + .context("failed to write to cryptsetup")?; + Ok(child + .wait() + .context("failed to wait for cryptsetup")? + .code()) +} + +/// Record that keyslot `slot` holds the key of `epoch` and `app_epoch`. +pub(super) fn write_luks_epochs(device: &Path, slot: u8, epoch: u32, app_epoch: u32) -> Result<()> { + let token = serde_json::json!({ + "type": ROOT_EPOCH_TOKEN_TYPE, + "keyslots": [slot.to_string()], + "root_epoch": epoch, + "app_epoch": app_epoch, + }); + let args = ["token", "import", "--token-id", "0", "--token-replace"]; + let code = run_with_input(cryptsetup(&args, device), token.to_string().as_bytes())?; + if code != Some(0) { + bail!("failed to record root epoch {epoch} in the LUKS header: {code:?}"); + } + Ok(()) +} + +/// Back up the LUKS header of `device` to `header` and validate the copy. +pub(super) fn load_luks_header(device: &Path, header: &Path) -> Result { + if header.exists() { + fs::remove_file(header)?; + } + let status = Command::new("cryptsetup") + .args(["luksHeaderBackup", "--header-backup-file"]) + .arg(header) + .arg(device) + .status() + .context("failed to start cryptsetup luksHeaderBackup")?; + if !status.success() { + bail!("failed to back up the LUKS header"); + } + validate_luks2_headers(fs::File::open(header)?).context("failed to validate the LUKS2 header") +} + +/// The keyslot of the validated `header` that `key` unlocks. +fn unlocked_keyslot(header: &Path, state: &LuksHeaderState, key: &str) -> Result> { + for &slot in &state.keyslots { + if unlocks(header, slot, key)? { + return Ok(Some(slot)); + } + } + Ok(None) +} + +/// Whether `key` unlocks keyslot `slot` of the validated `header`. +fn unlocks(header: &Path, slot: u8, key: &str) -> Result { + let slot_arg = slot.to_string(); + let test = cryptsetup( + &["open", "--test-passphrase", "--key-slot", &slot_arg, "-d-"], + header, + ); + match run_with_input(test, key.as_bytes())? { + Some(0) => Ok(true), + // No key available with this passphrase. + Some(2) => Ok(false), + code => bail!("failed to test the LUKS passphrase: {code:?}"), + } +} + +/// Add `new_key` in keyslot `slot`, authorized by `old_key`. The old key is +/// written next to `header`, which must be in a private in-memory directory. +fn add_luks_key( + device: &Path, + header: &Path, + old_key: &str, + new_key: &str, + slot: u8, +) -> Result<()> { + let old_file = header.with_file_name("old-key"); + defer! { + fs::remove_file(&old_file).ok(); + } + safe_write_with_mode(&old_file, old_key, 0o600)?; + let slot = slot.to_string(); + let args = [ + "luksAddKey", + "--batch-mode", + "--pbkdf", + "pbkdf2", + "--new-key-slot", + &slot, + ]; + let mut command = cryptsetup(&args, device); + command.arg("--key-file").arg(&old_file); + if run_with_input(command, new_key.as_bytes())? != Some(0) { + bail!("cryptsetup luksAddKey failed"); + } + Ok(()) +} + +fn kill_luks_slot(device: &Path, slot: u8) -> Result<()> { + let status = cryptsetup(&["luksKillSlot", "--batch-mode"], device) + .arg(slot.to_string()) + .stdin(Stdio::null()) + .status() + .context("failed to start cryptsetup luksKillSlot")?; + if !status.success() { + bail!("failed to remove LUKS keyslot {slot}"); + } + Ok(()) +} + +/// Bring the data disk to the `target` epochs, so that `key` unlocks its only +/// keyslot and the token records `target`. +/// +/// Each step is a single atomic LUKS2 metadata update: add the new key with +/// the old one, record the new epochs, then remove the old keyslot. Whatever +/// step a crash interrupts, the next boot finds either the old key alone or a +/// keyslot `key` already unlocks, and finishes from there. If another rotation +/// came first, `key` unlocks neither keyslot; the migration then restarts from +/// the keyslot the token records. `old_key` is only called when `key` unlocks +/// no keyslot. An unrotated epoch-0 disk is left +/// untouched, so it stays readable by older OS images. +/// +/// `header` holds the validated in-memory copy of `state` and is refreshed +/// after every change. +pub(super) async fn sync_luks_epochs( + device: &Path, + header: &Path, + mut state: LuksHeaderState, + key: &str, + target: (u32, u32), + old_key: F, +) -> Result +where + F: FnOnce(u32, u32) -> Fut, + Fut: std::future::Future>, +{ + let stored = (state.root_epoch.unwrap_or(0), state.app_epoch); + if stored.0 > target.0 || stored.1 > target.1 { + bail!("the data disk epochs {stored:?} exceed current {target:?}"); + } + let slot = match unlocked_keyslot(header, &state, key)? { + Some(slot) => slot, + None => { + if stored == target { + bail!("the data disk key does not unlock epochs {target:?}"); + } + // The keyslot holding the recorded epochs; a disk without a token + // is epoch 0 in slot 0. + let old_slot = state.token_keyslot.unwrap_or(0); + if !state.keyslots.contains(&old_slot) { + bail!("the recorded LUKS keyslot {old_slot} does not exist"); + } + info!("re-keying the data disk from epochs {stored:?} to {target:?}"); + let old_key = old_key(stored.0, stored.1).await?; + if !unlocks(header, old_slot, &old_key)? { + bail!( + "the key of epochs {stored:?} does not unlock the recorded keyslot {old_slot}" + ); + } + // A migration interrupted before a later rotation leaves a second + // keyslot whose epochs were never recorded, or are no longer the + // recorded ones. Drop it, now that the recorded key is confirmed, + // to make room for the current key. + for &stale in state.keyslots.iter().filter(|&&slot| slot != old_slot) { + kill_luks_slot(device, stale)?; + } + add_luks_key(device, header, &old_key, key, old_slot ^ 1)?; + state = load_luks_header(device, header)?; + unlocked_keyslot(header, &state, key)? + .context("the added LUKS keyslot does not unlock with the data disk key")? + } + }; + let mut changed = false; + let recorded = ( + state.token_keyslot, + state.root_epoch.unwrap_or(0), + state.app_epoch, + ); + let legacy = target == (0, 0) && state.token_keyslot.is_none(); + if !legacy && recorded != (Some(slot), target.0, target.1) { + write_luks_epochs(device, slot, target.0, target.1)?; + changed = true; + } + for &stale in state.keyslots.iter().filter(|&&stale| stale != slot) { + kill_luks_slot(device, stale)?; + changed = true; + } + if changed { + state = load_luks_header(device, header)?; + let recorded = (state.token_keyslot, state.root_epoch, state.app_epoch); + if state.keyslots != [slot] || recorded != (Some(slot), Some(target.0), target.1) { + bail!("the updated LUKS header does not record epochs {target:?}"); + } + } + Ok(state) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct LuksHeaderState { + /// Root epoch recorded in the dstack token; `None` means epoch 0. + pub(super) root_epoch: Option, + pub(super) app_epoch: u32, + /// Keyslot the dstack token records the epochs of. + pub(super) token_keyslot: Option, + /// Sorted keyslot ids; a second one exists only mid-migration. + pub(super) keyslots: Vec, +} + +pub(super) fn validate_luks2_headers(mut reader: impl std::io::Read) -> Result { + let primary = validate_single_luks2_header(&mut reader, 0)?; + let secondary = validate_single_luks2_header(&mut reader, 1)?; + if primary != secondary { + bail!("LUKS header copies disagree"); + } + Ok(primary) +} + +fn validate_single_luks2_header( + mut reader: impl std::io::Read, + hdr_ind: u64, +) -> Result { + let mut hdr_data = vec![0u8; BINARY_HEADER_SIZE as usize]; + reader + .read_exact(&mut hdr_data) + .context("Failed to read LUKS header")?; + let LuksHeader { + magic, + version, + hdr_size, + label, + csum_alg, + subsystem, + hdr_offset, + } = LuksHeader::parse(&hdr_data).context("Failed to decode LUKS header")?; + + let expected_magic = match hdr_ind { + 0 => [76, 85, 75, 83, 186, 190], + 1 => [83, 75, 85, 76, 186, 190], + _ => bail!("Invalid LUKS header index: {hdr_ind}"), + }; + if magic != expected_magic { + bail!("Invalid LUKS magic: {magic:?}"); + } + if version != 2 { + bail!("Invalid LUKS version: {version}"); + } + if label != [0; 48] { + bail!("Invalid LUKS label: {:?}", label); + } + if csum_alg != const_pad!(b"sha256", 32) { + bail!("Invalid LUKS checksum algorithm"); + } + if subsystem != [0; 48] { + bail!("Invalid LUKS subsystem"); + } + if hdr_offset != hdr_ind * hdr_size { + bail!("Invalid LUKS header offset: {hdr_offset}"); + } + if !(4096..=1024 * 1024 * 16).contains(&hdr_size) { + bail!("Invalid LUKS header size: {hdr_size}"); + } + + // Check JSON + let json_size = hdr_size - BINARY_HEADER_SIZE; + let mut jsn_data = vec![0u8; json_size as usize]; + reader + .read_exact(&mut jsn_data) + .context("Failed to read LUKS JSON")?; + let json_end = jsn_data + .iter() + .position(|&b| b == 0) + .unwrap_or(jsn_data.len()); + jsn_data.truncate(json_end); + + let LuksJson { + keyslots, + tokens, + segments, + digests, + config: + LuksConfig { + keyslots_size, + flags, + requirements, + }, + } = serde_json::from_slice(&jsn_data).context("Failed to decode LUKS JSON")?; + + let mut keyslot_ids: Vec = keyslots.keys().copied().collect(); + keyslot_ids.sort_unstable(); + if keyslot_ids != [0] && keyslot_ids != [1] && keyslot_ids != [0, 1] { + bail!("Invalid LUKS keyslots"); + } + let (root_epoch, app_epoch, token_keyslot) = match tokens.len() { + 0 => (None, 0, None), + 1 => { + let token = tokens.get(&0).context("Invalid LUKS token id")?; + let slot = match &token.keyslots[..] { + [slot] => slot.parse::().ok(), + _ => None, + }; + if token.kind != ROOT_EPOCH_TOKEN_TYPE + || !slot.is_some_and(|s| keyslots.contains_key(&s)) + { + bail!("Invalid LUKS token"); + } + let root_epoch = token.root_epoch.context("LUKS token has no root epoch")?; + (Some(root_epoch), token.app_epoch, slot) + } + _ => bail!("Invalid LUKS tokens"), + }; + if segments.len() != 1 { + bail!("Invalid LUKS segments"); + } + if digests.len() != 1 { + bail!("Invalid LUKS digests"); + } + if keyslots_size % 4096 != 0 { + bail!("Invalid LUKS keyslots size: {keyslots_size}"); + } + if flags.is_some() { + bail!("Invalid LUKS flags"); + } + if requirements.is_some() { + bail!("Invalid LUKS requirements"); + } + + for keyslot in keyslots.values() { + let LuksKeyslot::Luks2 { + key_size, + area: + LuksArea::Raw { + encryption, + key_size: area_key_size, + offset, + size, + }, + kdf, + af: + LuksAf::Luks1 { + hash: af_hash, + stripes, + }, + priority, + } = keyslot; + if encryption != "aes-xts-plain64" { + bail!("Invalid LUKS keyslot encryption: {encryption}"); + } + // Pin where the encrypted key material is read from. The binary area + // must sit between the two header copies and the encrypted payload; + // otherwise a host with raw disk access could redirect it elsewhere. + let area_end = offset.checked_add(*size); + if *offset < 2 * hdr_size || area_end.is_none_or(|end| end > PAYLOAD_OFFSET) { + bail!("Invalid LUKS keyslot area: offset={offset} size={size}"); + } + if *key_size != 64 { + bail!("Invalid LUKS keyslot key size: {key_size}"); + } + if *area_key_size != 64 { + bail!("Invalid LUKS keyslot key size: {area_key_size}"); + } + let LuksKdf::Pbkdf2 { hash: kdf_hash } = kdf else { + bail!("Invalid LUKS keyslot KDF"); + }; + if kdf_hash != "sha256" { + bail!("Invalid LUKS keyslot hash: {kdf_hash}"); + } + if af_hash != "sha256" { + bail!("Invalid LUKS keyslot hash: {af_hash}"); + } + if *stripes != 4000 { + bail!("Invalid LUKS keyslot stripes: {stripes}"); + } + if priority.is_some() { + bail!("Invalid LUKS keyslot priority"); + } + } + + { + let LuksSegment::Crypt { + offset, + size, + iv_tweak, + encryption, + sector_size, + integrity, + flags, + } = segments.get(&0).context("no LUKS segment")?; + if *offset != PAYLOAD_OFFSET { + bail!("Invalid LUKS segment offset"); + } + if size != "dynamic" { + bail!("Invalid LUKS segment size"); + } + if *iv_tweak != 0 { + bail!("Invalid LUKS segment IV tweak"); + } + if encryption != "aes-xts-plain64" { + bail!("Invalid LUKS segment encryption"); + } + if *sector_size != 512 { + bail!("Invalid LUKS segment sector size"); + } + if integrity.is_some() { + bail!("Invalid LUKS segment integrity"); + } + if flags.is_some() { + bail!("Invalid LUKS segment flags"); + } + } + { + let LuksDigest::Pbkdf2 { + keyslots, + segments, + hash, + } = digests.get(&0).context("no LUKS digest")?; + if hash != "sha256" { + bail!("Invalid LUKS digest hash: {hash}"); + } + let mut keyslots = keyslots.clone(); + keyslots.sort_unstable(); + if keyslots != keyslot_ids { + bail!("Invalid LUKS digest keyslots: {keyslots:?}"); + } + if segments != &[0] { + bail!("Invalid LUKS digest segments: {segments:?}"); + } + } + Ok(LuksHeaderState { + root_epoch, + app_epoch, + token_keyslot, + keyslots: keyslot_ids, + }) +} + +#[test] +fn test_validate_luks2_header() { + let header_data = include_bytes!("../../tests/fixtures/luks_header_good").to_vec(); + let state = + validate_luks2_headers(&mut &header_data[..]).expect("Failed to validate LUKS2 header"); + assert_eq!( + state, + LuksHeaderState { + root_epoch: None, + app_epoch: 0, + token_keyslot: None, + keyslots: vec![0], + } + ); + let header_data = include_bytes!("../../tests/fixtures/luks_header_cipher_null").to_vec(); + let error = validate_luks2_headers(&mut &header_data[..]).unwrap_err(); + assert!(error + .to_string() + .contains("Invalid LUKS keyslot encryption")); +} + +#[test] +fn test_validate_luks2_header_rejects_out_of_range_keyslot_area() { + // Redirect the keyslot binary area below the header region. Same length + // so the surrounding header stays intact; "00768" parses to 768, which is + // inside the header copies (< 2 * hdr_size) rather than the metadata gap. + let mut header = include_bytes!("../../tests/fixtures/luks_header_good").to_vec(); + let patched = patch_luks_json(&mut header, br#""offset":"32768""#, br#""offset":"00768""#); + assert_eq!(patched, 2, "expected to patch both header copies"); + let error = validate_luks2_headers(&mut &header[..]).unwrap_err(); + assert!(error.to_string().contains("Invalid LUKS keyslot area")); +} + +/// Replace `needle` with `replacement` inside every LUKS JSON region, keeping +/// each region the same length by giving back trailing NUL padding. +#[cfg(test)] +fn patch_luks_json(header: &mut [u8], needle: &[u8], replacement: &[u8]) -> usize { + assert!(replacement.len() >= needle.len()); + let grow = replacement.len() - needle.len(); + let mut patched = 0; + let mut i = 0; + while i + needle.len() <= header.len() { + if &header[i..i + needle.len()] != needle { + i += 1; + continue; + } + let tail = &mut header[i..]; + let end = tail.iter().position(|b| *b == 0).expect("NUL padding"); + assert!(end + grow < tail.len(), "not enough NUL padding"); + tail.copy_within(needle.len()..end, replacement.len()); + tail[..replacement.len()].copy_from_slice(replacement); + patched += 1; + i += replacement.len(); + } + patched +} + +#[cfg(test)] +struct TestDisk { + dir: tempfile::TempDir, + image: std::path::PathBuf, + header: std::path::PathBuf, +} + +#[cfg(test)] +impl TestDisk { + /// A disk formatted with `key` and recording `epochs`, or `None` without + /// cryptsetup. + fn new(key: &str, epochs: Option<(u32, u32)>) -> Option { + if Command::new("cryptsetup") + .arg("--version") + .output() + .is_err() + { + eprintln!("cryptsetup is unavailable; skipping"); + return None; + } + let dir = tempfile::tempdir().unwrap(); + let image = dir.path().join("disk.img"); + let header = dir.path().join("header"); + fs::File::create(&image) + .unwrap() + .set_len(2 * PAYLOAD_OFFSET) + .unwrap(); + let offset = (PAYLOAD_OFFSET / 512).to_string(); + let format = cryptsetup( + &[ + "luksFormat", + "--batch-mode", + "--type", + "luks2", + "--offset", + &offset, + "--cipher", + "aes-xts-plain64", + "--pbkdf", + "pbkdf2", + "--sector-size", + "512", + "-d-", + ], + &image, + ); + assert_eq!(run_with_input(format, key.as_bytes()).unwrap(), Some(0)); + if let Some((root, app)) = epochs { + write_luks_epochs(&image, 0, root, app).unwrap(); + } + Some(Self { dir, image, header }) + } + + fn state(&self) -> LuksHeaderState { + load_luks_header(&self.image, &self.header).unwrap() + } + + fn unlocks(&self, key: &str) -> bool { + let test = cryptsetup(&["open", "--test-passphrase", "-d-"], &self.image); + run_with_input(test, key.as_bytes()).unwrap() == Some(0) + } + + fn add_key(&self, old: &str, new: &str, slot: u8) { + add_luks_key(&self.image, &self.header, old, new, slot).unwrap(); + } + + /// Sync to `target`; `old` is the only key the KMS would hand out. + async fn sync( + &self, + key: &str, + target: (u32, u32), + old: Option<(u32, u32, &str)>, + ) -> Result { + let old_key = |root, app| { + let key = old + .filter(|(r, a, _)| (*r, *a) == (root, app)) + .map(|(_, _, key)| key.to_owned()) + .context("the KMS does not serve this epoch"); + std::future::ready(key) + }; + sync_luks_epochs( + &self.image, + &self.header, + self.state(), + key, + target, + old_key, + ) + .await + } + + fn assert_synced(&self, key: &str, target: (u32, u32), slot: u8) { + let state = self.state(); + assert_eq!(state.keyslots, [slot]); + assert_eq!(state.token_keyslot, Some(slot)); + assert_eq!( + (state.root_epoch, state.app_epoch), + (Some(target.0), target.1) + ); + assert!(self.unlocks(key)); + assert!(!self.dir.path().join("old-key").exists()); + } +} + +#[tokio::test] +async fn test_luks_epoch_migration() { + let Some(disk) = TestDisk::new("k1", Some((1, 0))) else { + return; + }; + disk.sync("k2", (2, 0), Some((1, 0, "k1"))).await.unwrap(); + disk.assert_synced("k2", (2, 0), 1); + assert!(!disk.unlocks("k1")); + disk.sync("k3", (2, 1), Some((2, 0, "k2"))).await.unwrap(); + disk.assert_synced("k3", (2, 1), 0); + assert!(!disk.unlocks("k2")); + // Already current: no key is requested. + disk.sync("k3", (2, 1), None).await.unwrap(); + disk.assert_synced("k3", (2, 1), 0); +} + +#[tokio::test] +async fn test_luks_epoch_migration_resumes_after_each_step() { + // Crash after luksAddKey, then after the token import as well. + for record in [false, true] { + let Some(disk) = TestDisk::new("k1", Some((1, 0))) else { + return; + }; + disk.add_key("k1", "k2", 1); + if record { + write_luks_epochs(&disk.image, 1, 2, 0).unwrap(); + } + disk.sync("k2", (2, 0), None).await.unwrap(); + disk.assert_synced("k2", (2, 0), 1); + assert!(!disk.unlocks("k1")); + } +} + +#[tokio::test] +async fn test_luks_epoch_migration_resumes_after_a_later_rotation() { + // Interrupted while migrating to epoch 1, then rotated to epoch 2 before + // the next boot: the current key unlocks neither keyslot. + for record in [false, true] { + let Some(disk) = TestDisk::new("k0", None) else { + return; + }; + disk.add_key("k0", "k1", 1); + let old = if record { + write_luks_epochs(&disk.image, 1, 1, 0).unwrap(); + (1, 0, "k1") + } else { + (0, 0, "k0") + }; + disk.sync("k2", (2, 0), Some(old)).await.unwrap(); + disk.assert_synced("k2", (2, 0), if record { 0 } else { 1 }); + assert!(!disk.unlocks("k0")); + assert!(!disk.unlocks("k1")); + } +} + +#[tokio::test] +async fn test_luks_stale_keyslot_is_kept_until_the_recorded_key_is_confirmed() { + let Some(disk) = TestDisk::new("k0", None) else { + return; + }; + disk.add_key("k0", "k1", 1); + let interrupted = disk.state(); + // The KMS denies the recorded epoch, or hands out a key that does not + // unlock its keyslot: both keyslots stay. + assert!(disk.sync("k2", (2, 0), None).await.is_err()); + let err = disk + .sync("k2", (2, 0), Some((0, 0, "k1"))) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("does not unlock the recorded keyslot 0"), + "{err:#}" + ); + assert_eq!(disk.state(), interrupted); + assert!(disk.unlocks("k0") && disk.unlocks("k1")); +} + +#[tokio::test] +async fn test_luks_epoch_migration_with_disagreeing_header_copies() { + let Some(disk) = TestDisk::new("k1", Some((1, 0))) else { + return; + }; + disk.add_key("k1", "k2", 1); + let before = fs::read(&disk.image).unwrap(); + write_luks_epochs(&disk.image, 1, 2, 0).unwrap(); + let after = fs::read(&disk.image).unwrap(); + let hdr_size = LuksHeader::parse(&before).unwrap().hdr_size as usize; + // Interrupted between writing the two header copies, in either order. + for (primary, secondary) in [(&after, &before), (&before, &after)] { + let mut image = primary.clone(); + image[hdr_size..2 * hdr_size].copy_from_slice(&secondary[hdr_size..2 * hdr_size]); + fs::write(&disk.image, image).unwrap(); + disk.sync("k2", (2, 0), None).await.unwrap(); + disk.assert_synced("k2", (2, 0), 1); + } +} + +#[tokio::test] +async fn test_luks_epoch_migration_failures_keep_the_old_epochs() { + let Some(disk) = TestDisk::new("k1", Some((1, 0))) else { + return; + }; + let unchanged = disk.state(); + // The KMS denies the old epoch. + assert!(disk.sync("k2", (2, 0), None).await.is_err()); + // luksAddKey fails: the KMS answers with a key that does not unlock. + assert!(disk.sync("k2", (2, 0), Some((1, 0, "bad"))).await.is_err()); + // The disk records a root epoch newer than the current one. + let err = disk.sync("k0", (0, 3), None).await.unwrap_err(); + assert!(err.to_string().contains("exceed"), "{err:#}"); + assert_eq!(disk.state(), unchanged); + assert!(disk.unlocks("k1")); + + let app_ahead = TestDisk::new("k1", Some((1, 2))).unwrap(); + let err = app_ahead.sync("k1", (1, 1), None).await.unwrap_err(); + assert!(err.to_string().contains("exceed"), "{err:#}"); + + // A failed token import is an error. + let not_luks = disk.dir.path().join("not-luks"); + fs::write(¬_luks, [0; 4096]).unwrap(); + assert!(write_luks_epochs(¬_luks, 0, 2, 0).is_err()); +} + +#[tokio::test] +async fn test_luks_epoch_zero_disk_is_left_untouched() { + let Some(disk) = TestDisk::new("k0", None) else { + return; + }; + let unchanged = disk.state(); + disk.sync("k0", (0, 0), None).await.unwrap(); + assert_eq!(disk.state(), unchanged); + assert_eq!(unchanged.token_keyslot, None); +} + +#[test] +fn test_validate_luks2_header_root_epoch_token() { + let with_token = |token: &str| { + let mut header = include_bytes!("../../tests/fixtures/luks_header_good").to_vec(); + let replacement = format!(r#""tokens":{{"0":{token}}}"#); + let patched = patch_luks_json(&mut header, br#""tokens":{}"#, replacement.as_bytes()); + assert_eq!(patched, 2, "expected to patch both header copies"); + validate_luks2_headers(&mut &header[..]) + }; + let state = with_token(r#"{"type":"dstack-kms","keyslots":["0"],"root_epoch":2}"#).unwrap(); + assert_eq!(state.root_epoch, Some(2)); + for token in [ + r#"{"type":"luks2-keyring","keyslots":["0"],"root_epoch":2}"#, + r#"{"type":"dstack-kms","keyslots":["1"],"root_epoch":2}"#, + r#"{"type":"dstack-kms","keyslots":["0"]}"#, + ] { + assert!(with_token(token).is_err(), "{token}"); + } +} + +#[test] +fn test_validate_luks2_header_rejects_keyslot_area_that_overflows() { + let mut header = include_bytes!("../../tests/fixtures/luks_header_good").to_vec(); + // 2**64 - 32768 + 1000: added to the accepted offset 32768 it wraps to + // 1000, which is below PAYLOAD_OFFSET. + let patched = patch_luks_json( + &mut header, + br#""size":"258048""#, + br#""size":"18446744073709519848""#, + ); + assert_eq!(patched, 2, "expected to patch both header copies"); + let error = validate_luks2_headers(&mut &header[..]).unwrap_err(); + assert!( + error.to_string().contains("Invalid LUKS keyslot area"), + "{error:#}" + ); +} diff --git a/dstack-util/src/system_setup/sealed_env.rs b/dstack-util/src/system_setup/sealed_env.rs new file mode 100644 index 000000000..ef7a717f6 --- /dev/null +++ b/dstack-util/src/system_setup/sealed_env.rs @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Environment envelope encoding and encryption for guest boot migration. + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub(super) struct SealedEnv { + pub(super) version: u32, + pub(super) root_epoch: u32, + pub(super) app_epoch: u32, + #[serde(with = "serde_human_bytes::base64")] + pub(super) enc: Vec, + #[serde(with = "serde_human_bytes::base64")] + pub(super) ciphertext: Vec, +} + +pub(super) fn sealed_env_info_at_epochs(root: u32, app: u32) -> Vec { + format!("dstack-sealed-env-v1:{root}:{app}").into_bytes() +} + +pub(super) fn seal_env_at_epochs( + secret: &[u8], + epoch: u32, + app_epoch: u32, + plaintext: &[u8], +) -> Result> { + let secret: [u8; 32] = secret.try_into().context("invalid sealed env key")?; + let public = x25519_dalek::PublicKey::from(&x25519_dalek::StaticSecret::from(secret)); + let payload = crate::crypto::sealed_env_encrypt( + &public.to_bytes(), + &sealed_env_info_at_epochs(epoch, app_epoch), + plaintext, + )?; + let (enc, ciphertext) = payload.split_at(32); + Ok(serde_json::to_vec(&SealedEnv { + version: 1, + root_epoch: epoch, + app_epoch, + enc: enc.to_vec(), + ciphertext: ciphertext.to_vec(), + })?) +} + +pub(super) fn env_crypt_key(key: &[u8]) -> Result<[u8; 32]> { + key.try_into() + .ok() + .context("encrypted env is present but the key provider gave no valid env crypt key") +} + +#[cfg(test)] +pub(super) fn seal_env(secret: &[u8], epoch: u32, plaintext: &[u8]) -> Result> { + seal_env_at_epochs(secret, epoch, 0, plaintext) +} diff --git a/dstack-util/tests/fixtures/sealed-env-javascript.json b/dstack-util/tests/fixtures/sealed-env-javascript.json new file mode 100644 index 000000000..71345c70e --- /dev/null +++ b/dstack-util/tests/fixtures/sealed-env-javascript.json @@ -0,0 +1,18 @@ +{ + "description": "Synthetic public test fixture produced by @hpke/core 1.10.0; never use its key in a deployment.", + "private_key": "0707070707070707070707070707070707070707070707070707070707070707", + "public_key": "13be4feaeaf204c7fd3358fc9c00721881d174278128227ec674f37f7fe97b6d", + "env": [ + { + "key": "MESSAGE", + "value": "Hello 世界" + } + ], + "sealed_env": { + "version": 1, + "root_epoch": 3, + "app_epoch": 0, + "enc": "UDky3L18JX+ER+SvKjS84lCImL7hB1c384BHvYD5Y3M=", + "ciphertext": "kocgwHn2LYl16jj08iIT5e7BjaOtqDlDYBuFAJLa81MF4Gg9fnvSBqQteSGSLNWP3vKt2W6FOv9mMqJlHchFoioP" + } +} diff --git a/dstack-util/tests/fixtures/sealed-env-python.json b/dstack-util/tests/fixtures/sealed-env-python.json new file mode 100644 index 000000000..3e28366c9 --- /dev/null +++ b/dstack-util/tests/fixtures/sealed-env-python.json @@ -0,0 +1,18 @@ +{ + "description": "Synthetic public test fixture produced by pyhpke 0.6.5; never use its key in a deployment.", + "private_key": "0707070707070707070707070707070707070707070707070707070707070707", + "public_key": "13be4feaeaf204c7fd3358fc9c00721881d174278128227ec674f37f7fe97b6d", + "env": [ + { + "key": "MESSAGE", + "value": "Hello 世界" + } + ], + "sealed_env": { + "version": 1, + "root_epoch": 3, + "app_epoch": 0, + "enc": "8PU8Zzkd7dsVGa48MjwyOEchWjKTI1s1lbDcm03Wj2I=", + "ciphertext": "V4xHwrD3JYdRHJNMRCT10xSoEpMEOO8KXNuI8SQz49u1CraXsBSoIpDUAgajkIRqPSgwMfNL5ONyRtx8kQ10gDe6XGRsugFOMRxHTQ==" + } +} diff --git a/guest-agent/src/rpc_service.rs b/guest-agent/src/rpc_service.rs index 4e61aa240..bb5429d03 100644 --- a/guest-agent/src/rpc_service.rs +++ b/guest-agent/src/rpc_service.rs @@ -912,6 +912,7 @@ pNs85uhOZE8z2jr8Pg== let dummy_keys = AppKeys { disk_crypt_key: Vec::new(), env_crypt_key: Vec::new(), + sealed_env_key: Vec::new(), k256_key: DUMMY_K256_KEY.to_vec(), k256_signature: Vec::new(), gateway_app_id: String::new(), diff --git a/host-api/proto/host_api.proto b/host-api/proto/host_api.proto index f8864f516..85106ed6e 100644 --- a/host-api/proto/host_api.proto +++ b/host-api/proto/host_api.proto @@ -27,8 +27,19 @@ message GetSealingKeyResponse { bytes provider_quote = 2; } +// The env the guest re-encrypted to the current KMS root epoch at boot; empty +// when it needed no change. +message SealedEnv { + bytes sealed_env = 1; +} + service HostApi { rpc Info(google.protobuf.Empty) returns (HostInfo); rpc Notify(Notification) returns (google.protobuf.Empty); rpc GetSealingKey(GetSealingKeyRequest) returns (GetSealingKeyResponse); + // Close this boot's env update window, replacing the VM's stored encrypted + // env if it is still the one the VM booted with. Accepted once per boot; any + // answer means the window is closed, and the guest starts application code + // only after one. + rpc UpdateSealedEnv(SealedEnv) returns (google.protobuf.Empty); } diff --git a/host-api/src/client.rs b/host-api/src/client.rs index 5a0606009..aa6f214ff 100644 --- a/host-api/src/client.rs +++ b/host-api/src/client.rs @@ -4,6 +4,7 @@ use crate::host_api_client::HostApiClient; use http_client::prpc::PrpcClient; +pub use http_client::prpc::StatusError; pub type DefaultClient = HostApiClient; diff --git a/http-client/src/prpc.rs b/http-client/src/prpc.rs index eacc03b1e..3b63b8989 100644 --- a/http-client/src/prpc.rs +++ b/http-client/src/prpc.rs @@ -33,6 +33,53 @@ impl PrpcClient { } } +/// A pRPC server answered with a status other than 200. +/// +/// Carried in the returned `anyhow::Error`, so callers can tell, say, a method +/// the server does not implement from other failures without parsing text. +#[derive(Debug)] +pub struct StatusError { + pub path: String, + pub status: u16, + /// The reason the server gave, if it gave one. + pub message: Option, +} + +impl std::fmt::Display for StatusError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Self { + path, + status, + message, + } = self; + match message { + Some(message) => write!(f, "{path} failed with status {status}: {message}"), + None => write!(f, "Invalid status code: {status}, path={path}"), + } + } +} + +impl std::error::Error for StatusError {} + +/// The reason a pRPC server gave for failing a JSON request, if it gave one. +fn server_error_message(body: &[u8]) -> Option { + #[derive(serde::Deserialize)] + struct ErrorBody { + error: String, + } + serde_json::from_slice::(body) + .ok() + .map(|body| body.error) +} + +fn normalize_json_response_body(body: &[u8]) -> &[u8] { + if body.is_empty() { + b"null" + } else { + body + } +} + impl RequestClient for PrpcClient { async fn request(&self, path: &str, body: T) -> Result where @@ -43,9 +90,15 @@ impl RequestClient for PrpcClient { let path = format!("{}{path}?json", self.path_append); let (status, body) = super::http_request("POST", &self.base_url, &path, &body).await?; if status != 200 { - anyhow::bail!("Invalid status code: {status}, path={path}"); + return Err(StatusError { + path, + status, + message: server_error_message(&body), + } + .into()); } - let response = serde_json::from_slice(&body).context("Failed to deserialize response")?; + let response = serde_json::from_slice(normalize_json_response_body(&body)) + .context("Failed to deserialize response")?; Ok(response) } } diff --git a/kms/rpc/proto/kms_rpc.proto b/kms/rpc/proto/kms_rpc.proto index 9ef55b98e..f7de032fd 100644 --- a/kms/rpc/proto/kms_rpc.proto +++ b/kms/rpc/proto/kms_rpc.proto @@ -11,6 +11,10 @@ package kms; message GetAppKeyRequest { uint32 api_version = 1; string vm_config = 2; + // Root key epoch to derive from; unset means the current epoch. Epoch 0 is + // the original root keys. Older epochs remain available so CVMs can migrate. + optional uint32 root_epoch = 3; + optional uint32 app_epoch = 4; } message AppId { @@ -46,6 +50,11 @@ message AppKeyResponse { string gateway_app_id = 7; // OS Image hash bytes os_image_hash = 8; + // Root key epoch all epoch-dependent fields above were derived from. + uint32 root_epoch = 9; + // X25519 private key for the SealedEnv HPKE receiver, derived for this epoch. + bytes sealed_env_key = 10; + uint32 app_epoch = 11; } message GetMetaResponse { diff --git a/kms/src/main_service.rs b/kms/src/main_service.rs index 00723566b..27a9a9389 100644 --- a/kms/src/main_service.rs +++ b/kms/src/main_service.rs @@ -335,6 +335,9 @@ impl KmsRpc for RpcHandler { tproxy_app_id: gateway_app_id.clone(), gateway_app_id, os_image_hash: boot_info.os_image_hash, + root_epoch: 0, + sealed_env_key: vec![], + app_epoch: 0, }) } diff --git a/vmm/src/host_api_service.rs b/vmm/src/host_api_service.rs index fc4dcc704..6739f07f0 100644 --- a/vmm/src/host_api_service.rs +++ b/vmm/src/host_api_service.rs @@ -5,10 +5,11 @@ use anyhow::{bail, Context, Result}; use host_api::{ host_api_server::{HostApiRpc, HostApiServer}, - GetSealingKeyRequest, GetSealingKeyResponse, HostInfo, Notification, + GetSealingKeyRequest, GetSealingKeyResponse, HostInfo, Notification, SealedEnv, }; use ra_rpc::{CallContext, RemoteEndpoint, RpcCall}; use rocket_vsock_listener::VsockEndpoint; +use tracing::warn; use crate::app::App; use key_provider_client::host::get_key; @@ -60,4 +61,16 @@ impl HostApiRpc for HostApiHandler { provider_quote: response.provider_quote, }) } + + async fn update_sealed_env(self, request: SealedEnv) -> Result<()> { + // This VMM has no env update window to close, and its images take no + // sealed env, so there is nothing to store. + if !request.sealed_env.is_empty() { + warn!( + cid = self.endpoint.cid, + "ignoring a sealed env update from the guest" + ); + } + Ok(()) + } }