From ddabb56a9ca5f36ce34aa3e2bdaf0f61759093f2 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 5 Nov 2026 23:30:11 -0800 Subject: [PATCH] feat(gateway): support Google Public CA account registration with EAB Signed-off-by: Kevin Wang --- docs/certbot-dns-persist-01.md | 73 ++++++++++- dstack/Cargo.lock | 1 + dstack/certbot/Cargo.toml | 1 + dstack/certbot/src/acme_client.rs | 140 ++++++++++++++++++++- dstack/certbot/src/lib.rs | 1 + dstack/gateway/README.md | 3 +- dstack/gateway/rpc/proto/gateway_rpc.proto | 11 +- dstack/gateway/src/admin_service.rs | 88 +++++++++++-- dstack/gateway/src/distributed_certbot.rs | 20 +-- dstack/gateway/src/main_service.rs | 3 +- 10 files changed, 318 insertions(+), 23 deletions(-) diff --git a/docs/certbot-dns-persist-01.md b/docs/certbot-dns-persist-01.md index c3b264313..955b57a31 100644 --- a/docs/certbot-dns-persist-01.md +++ b/docs/certbot-dns-persist-01.md @@ -25,7 +25,8 @@ Cloudflare account, no API token, no provider integration. > Let's Encrypt has said it will not deploy the challenge to production until > that is resolved. It is live on Let's Encrypt **staging** and in > [Pebble][pebble]. Treat the record format as unstable, and expect to -> republish when the draft settles. +> republish when the draft settles. Google Trust Services also supports draft-01 +> in production; see [Google Public CA](#google-public-ca) below. [draft]: https://datatracker.ietf.org/doc/html/draft-ietf-acme-dns-persist-01 [pebble]: https://github.com/letsencrypt/pebble @@ -181,6 +182,76 @@ script, an older SDK — cannot downgrade a `dns-persist-01` domain to `dns-01` editing something else, which its hand-published CAA would then refuse. Send the field explicitly to switch a domain over. +### Google Public CA + +Google Public CA can issue certificates for gateways hosted outside Google Cloud. +A Google Cloud project is required to register the ACME account, not to host the +service. GTS supports `dns-persist-01` draft-01 in production as of September 28, +2026; the protocol is still experimental and a final standard may require DNS +record changes. + +1. Create or select a Google Cloud project, enable `publicca.googleapis.com`, and + obtain the `roles/publicca.externalAccountKeyCreator` role. Follow the + [Public CA tutorial][gts-tutorial] to create an external account binding (EAB) + key. Treat its HMAC value as a secret: do not paste it into logs, shell history, + or issue reports. +2. Call `Admin.SetCertbotConfig` with: + + ```json + { + "acme_url": "https://dv.acme-v02.api.pki.goog/directory", + "issuer_domain_name": "pki.goog" + } + ``` + + For testing, use `https://dv.acme-v02.test-api.pki.goog/directory` instead. + Staging certificates are not publicly trusted; use an EAB key created for the + corresponding environment as described in the tutorial. +3. Before adding domains, call the authenticated admin RPC + `Admin.RotateAcmeCredentials` with a request body containing: + + ```json + { + "eab_kid": "", + "eab_hmac_key": "" + } + ``` + + Send the real values through a protected request body over a trusted channel, + not as command-line arguments. Both fields are required when using EAB; + padded and unpadded base64url are accepted. The gateway uses them only for this + registration, never stores them in KV, and never returns them. Existing + clients and CAs without EAB can still send `{}`. +4. Add the ZT domain with `challenge: "dns-persist-01"`, then publish the exact + `required_dns_records` returned by the gateway. For example: + + ```dns + _validation-persist.app.example.com. IN TXT "pki.goog; accounturi=; policy=wildcard" + ``` + + Update existing CAA restrictions as well: they must allow `pki.goog`, the new + account URI, and `dns-persist-01`, rather than pinning issuance to the previous + CA or account. Renew the domain certificate once DNS has propagated. + +Each EAB key can register only one account. Renewals reuse the stored ACME +account and need neither EAB nor Google Cloud credentials. A later account +rotation requires a fresh EAB key and republishing persistent TXT and CAA +records. Every rotation creates a new account; do not repeat it merely because +manual DNS updates or CAA re-pinning are still outstanding. + +The ACME directory and issuer are global to the cluster: switching an existing +cluster from Let's Encrypt affects every ZT domain, not just newly added ones. +For `dns-01`, the same EAB registration flow works, but domains still need their +DNS provider credentials. The dashboard's register-account button sends no EAB; +use the admin RPC above for Google account registration and rotation. + +Google's default `newOrder` quota is 100 requests/hour per project, shared by all +ACME accounts in that project. Stagger bulk issuance and renewals, and respect +`Retry-After` on rate-limited responses. See [Public CA quotas][gts-quotas]. + +[gts-tutorial]: https://docs.cloud.google.com/certificate-manager/docs/public-ca-tutorial +[gts-quotas]: https://docs.cloud.google.com/certificate-manager/docs/quotas#public_ca_request_quotas + ### Naming the CA For a non-production ACME server, set `issuer_domain_name` in the global certbot diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index f4ff68f9a..3d3d10e74 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -847,6 +847,7 @@ name = "certbot" version = "0.6.3" dependencies = [ "anyhow", + "base64 0.22.1", "bon", "bytes", "enum_dispatch", diff --git a/dstack/certbot/Cargo.toml b/dstack/certbot/Cargo.toml index 6e76ddb96..af855f78e 100644 --- a/dstack/certbot/Cargo.toml +++ b/dstack/certbot/Cargo.toml @@ -31,6 +31,7 @@ tracing.workspace = true x509-parser.workspace = true [dev-dependencies] +base64.workspace = true rand.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["full"] } diff --git a/dstack/certbot/src/acme_client.rs b/dstack/certbot/src/acme_client.rs index 3406884ca..a5e3ba6cd 100644 --- a/dstack/certbot/src/acme_client.rs +++ b/dstack/certbot/src/acme_client.rs @@ -10,7 +10,7 @@ use hickory_resolver::proto::rr::RData; use hickory_resolver::TokioResolver; use instant_acme::{ Account, AccountCredentials, AuthorizationStatus, AuthorizedIdentifier, ChallengeType, - Identifier, NewAccount, NewOrder, Order, OrderStatus, Problem, + ExternalAccountKey, Identifier, NewAccount, NewOrder, Order, OrderStatus, Problem, }; use rcgen::{CertificateParams, DistinguishedName, KeyPair}; use serde::{Deserialize, Serialize}; @@ -418,7 +418,16 @@ impl AcmeClient { /// is forced under `dns-persist-01`: the record an operator publishes names /// the account, so the account has to exist first. pub async fn register_account(acme_url: &str) -> Result { - let (_, credentials) = Self::register(acme_url).await?; + Self::register_account_with_eab(acme_url, None).await + } + + /// Register an account using optional RFC 8555 external account binding. + /// The binding key is used only for registration and is not persisted. + pub async fn register_account_with_eab( + acme_url: &str, + external_account: Option<&ExternalAccountKey>, + ) -> Result { + let (_, credentials) = Self::register(acme_url, external_account).await?; Ok(AcmeAccount { account_uri: credentials.account_id.clone(), credentials: serde_json::to_string(&credentials)?, @@ -436,7 +445,7 @@ impl AcmeClient { // transient failure there would drop an account that exists at the CA // and counts against its registration limit, on a path whose caller has // not persisted anything yet. - let (account, credentials) = Self::register(acme_url).await?; + let (account, credentials) = Self::register(acme_url, None).await?; Ok(Self { account, validation, @@ -447,7 +456,10 @@ impl AcmeClient { /// Register an account and keep both halves: the live account and the /// credentials that reconstruct it. - async fn register(acme_url: &str) -> Result<(Account, Credentials)> { + async fn register( + acme_url: &str, + external_account: Option<&ExternalAccountKey>, + ) -> Result<(Account, Credentials)> { let http_client = Box::new(ReqwestHttpClient::new()?); let (account, credentials) = Account::builder_with_http(http_client) .create( @@ -457,7 +469,7 @@ impl AcmeClient { only_return_existing: false, }, acme_url.to_string(), - None, + external_account, ) .await .with_context(|| format!("failed to create ACME account for {acme_url}"))?; @@ -2090,3 +2102,121 @@ mod publish_tests { ); } } + +#[cfg(test)] +mod eab_tests { + use super::*; + use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + #[tokio::test] + async fn registration_sends_optional_eab_without_persisting_it() { + for with_eab in [false, true] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let server_base = base.clone(); + let server = tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut data = Vec::new(); + let header_end = loop { + let mut buf = [0; 4096]; + let n = stream.read(&mut buf).await.unwrap(); + assert_ne!(n, 0); + data.extend_from_slice(&buf[..n]); + if let Some(end) = data.windows(4).position(|w| w == b"\r\n\r\n") { + break end + 4; + } + }; + let headers = String::from_utf8(data[..header_end].to_vec()).unwrap(); + let content_length: usize = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse().unwrap()) + }) + .unwrap_or(0); + while data.len() < header_end + content_length { + let mut buf = [0; 4096]; + let n = stream.read(&mut buf).await.unwrap(); + assert_ne!(n, 0); + data.extend_from_slice(&buf[..n]); + } + let is_account = headers.starts_with("POST /account "); + let body = if headers.starts_with("GET /directory ") { + serde_json::json!({ + "newNonce": format!("{server_base}/nonce"), + "newAccount": format!("{server_base}/account"), + "newOrder": format!("{server_base}/order") + }) + .to_string() + } else if is_account { + let jws: serde_json::Value = + serde_json::from_slice(&data[header_end..]).unwrap(); + let payload: serde_json::Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(jws["payload"].as_str().unwrap()) + .unwrap(), + ) + .unwrap(); + let binding = &payload["externalAccountBinding"]; + assert_eq!(!binding.is_null(), with_eab); + if with_eab { + let protected: serde_json::Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(binding["protected"].as_str().unwrap()) + .unwrap(), + ) + .unwrap(); + assert_eq!(protected["kid"], "test-eab-id"); + assert_eq!(protected["alg"], "HS256"); + assert_eq!(protected["url"], format!("{server_base}/account")); + let outer: serde_json::Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(jws["protected"].as_str().unwrap()) + .unwrap(), + ) + .unwrap(); + let bound_key: serde_json::Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(binding["payload"].as_str().unwrap()) + .unwrap(), + ) + .unwrap(); + assert_eq!(bound_key, outer["jwk"]); + assert!(!binding["signature"].as_str().unwrap().is_empty()); + } + "{}".to_string() + } else { + assert!(headers.starts_with("HEAD /nonce ")); + String::new() + }; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nReplay-Nonce: test-nonce\r\nLocation: {server_base}/acct/1\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len() + ); + stream.write_all(response.as_bytes()).await.unwrap(); + if is_account { + break; + } + } + }); + let key = ExternalAccountKey::new("test-eab-id".into(), b"synthetic-test-hmac-key"); + let account = tokio::time::timeout( + Duration::from_secs(5), + AcmeClient::register_account_with_eab( + &format!("{base}/directory"), + with_eab.then_some(&key), + ), + ) + .await + .unwrap() + .unwrap(); + server.await.unwrap(); + assert_eq!(account.account_uri, format!("{base}/acct/1")); + assert!(!account.credentials.contains("test-eab-id")); + assert!(!account.credentials.contains("synthetic-test-hmac-key")); + } + } +} diff --git a/dstack/certbot/src/lib.rs b/dstack/certbot/src/lib.rs index e2a0314c9..284b3cbd2 100644 --- a/dstack/certbot/src/lib.rs +++ b/dstack/certbot/src/lib.rs @@ -24,6 +24,7 @@ pub use acme_client::{ pub use bot::{read_pubkey, CertBot, CertBotConfig}; pub use dns01_client::Dns01Client; pub use dns_persist::{resolve_issuer_domain_name, LETS_ENCRYPT_ISSUER_DOMAIN_NAME}; +pub use instant_acme::ExternalAccountKey; pub use workdir::WorkDir; mod acme_client; diff --git a/dstack/gateway/README.md b/dstack/gateway/README.md index af79ee75f..4bf3faf6f 100644 --- a/dstack/gateway/README.md +++ b/dstack/gateway/README.md @@ -1,7 +1,7 @@ # dstack-gateway The reverse proxy that gives deployed dstack apps zero-trust network access. It -terminates TLS, provisions certificates over ACME DNS-01, and routes traffic to +terminates TLS, provisions certificates over ACME DNS-01 or DNS-PERSIST-01, and routes traffic to CVMs across a WireGuard mesh it manages itself. ## Ingress mapping @@ -37,6 +37,7 @@ The gateway needs `CAP_NET_ADMIN` to manage its WireGuard interface, hence ## Further reading +- [Certificate provisioning](../../docs/certbot-dns-persist-01.md) — DNS-PERSIST-01 and Google Public CA - [Cluster deployment](docs/cluster-deployment.md) — running a replicated gateway - [Builder image](dstack-app/builder/README.md) — reproducible container image - [Deployment guide](../../docs/deployment.md) diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto index 04ad4c795..bf7f223bd 100644 --- a/dstack/gateway/rpc/proto/gateway_rpc.proto +++ b/dstack/gateway/rpc/proto/gateway_rpc.proto @@ -214,6 +214,14 @@ message AcmeInfoResponse { string account_attestation = 5; } +// Optional one-time external account binding for account registration. +// Omit both fields for CAs that do not require EAB. Never persisted or returned. +message RotateAcmeCredentialsRequest { + string eab_kid = 1; + // Base64url-encoded HMAC key, with or without padding. + string eab_hmac_key = 2; +} + // Result of replacing the shared ACME account credentials. message RotateAcmeCredentialsResponse { // URI of the newly-created ACME account. The private credentials are never returned. @@ -579,7 +587,8 @@ service Admin { // nodes by a best-effort lock (WaveKV has no compare-and-swap), so still // avoid rotating from multiple gateways concurrently. This re-pins issuance // to the new account; it does not deactivate the old ACME account at the CA. - rpc RotateAcmeCredentials(google.protobuf.Empty) returns (RotateAcmeCredentialsResponse) {} + // Optional EAB credentials are used only to register the new account. + rpc RotateAcmeCredentials(RotateAcmeCredentialsRequest) returns (RotateAcmeCredentialsResponse) {} // ==================== Tombstone GC Configuration ==================== // Get the effective tombstone GC pacing on this node diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs index 1ce6b7810..0d9ae49d1 100644 --- a/dstack/gateway/src/admin_service.rs +++ b/dstack/gateway/src/admin_service.rs @@ -20,11 +20,12 @@ use dstack_gateway_rpc::{ ListZtDomainsResponse, NodeStatusEntry, PeerSyncStatus as ProtoPeerSyncStatus, PortAttrs as RpcPortAttrs, PortPolicy as RpcPortPolicy, RejectedInstanceInfo, RemoveCvmRequest, RemoveCvmResponse, RemoveNodeRequest, RemoveNodeResponse, RenewCertResponse, - RenewZtDomainCertRequest, RenewZtDomainCertResponse, RotateAcmeCredentialsResponse, - SetCertbotConfigRequest, SetDefaultDnsCredentialRequest, SetInstancePortPolicyRequest, - SetInstanceReadyRequest, SetNodeStatusRequest, SetNodeUrlRequest, SetTombstoneGcConfigRequest, - StatusResponse, StoreSyncStatus, TombstoneGcConfigResponse, UpdateDnsCredentialRequest, - WaveKvStatusResponse, ZtDomainCertStatus, ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo, + RenewZtDomainCertRequest, RenewZtDomainCertResponse, RotateAcmeCredentialsRequest, + RotateAcmeCredentialsResponse, SetCertbotConfigRequest, SetDefaultDnsCredentialRequest, + SetInstancePortPolicyRequest, SetInstanceReadyRequest, SetNodeStatusRequest, SetNodeUrlRequest, + SetTombstoneGcConfigRequest, StatusResponse, StoreSyncStatus, TombstoneGcConfigResponse, + UpdateDnsCredentialRequest, WaveKvStatusResponse, ZtDomainCertStatus, + ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo, }; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; @@ -42,6 +43,32 @@ use crate::{ time::now_secs, }; +fn external_account_key( + request: &RotateAcmeCredentialsRequest, +) -> Result> { + use base64::{ + engine::general_purpose::{URL_SAFE, URL_SAFE_NO_PAD}, + Engine, + }; + + if request.eab_kid.is_empty() && request.eab_hmac_key.is_empty() { + return Ok(None); + } + ensure!( + !request.eab_kid.is_empty() && !request.eab_hmac_key.is_empty(), + "EAB requires both eab_kid and eab_hmac_key" + ); + let key = URL_SAFE_NO_PAD + .decode(&request.eab_hmac_key) + .or_else(|_| URL_SAFE.decode(&request.eab_hmac_key)) + .context("eab_hmac_key must be base64url encoded")?; + ensure!(!key.is_empty(), "EAB HMAC key must not be empty"); + Ok(Some(certbot::ExternalAccountKey::new( + request.eab_kid.clone(), + &key, + ))) +} + pub struct AdminRpcHandler { state: Proxy, } @@ -125,8 +152,15 @@ impl AdminRpc for AdminRpcHandler { self.state.reload_all_certs_from_kvstore() } - async fn rotate_acme_credentials(self) -> Result { - let outcome = self.state.rotate_acme_credentials().await?; + async fn rotate_acme_credentials( + self, + request: RotateAcmeCredentialsRequest, + ) -> Result { + let external_account = external_account_key(&request)?; + let outcome = self + .state + .rotate_acme_credentials(external_account.as_ref()) + .await?; Ok(RotateAcmeCredentialsResponse { account_uri: outcome.account_uri, domains_updated: outcome.domains_updated.try_into().unwrap_or(u32::MAX), @@ -1552,3 +1586,43 @@ mod cert_lock_admin_tests { ); } } + +#[cfg(test)] +mod eab_tests { + use super::*; + + #[test] + fn optional_eab_accepts_padded_and_unpadded_base64url() { + assert!(external_account_key( + &serde_json::from_str::("{}").unwrap() + ) + .unwrap() + .is_none()); + for encoded in ["-_8", "-_8="] { + let request = RotateAcmeCredentialsRequest { + eab_kid: "test-key-id".into(), + eab_hmac_key: encoded.into(), + }; + assert!(external_account_key(&request).unwrap().is_some()); + } + } + + #[test] + fn invalid_eab_is_rejected_without_echoing_secrets() { + for (kid, key) in [ + ("kid", ""), + ("", "test-secret"), + ("kid", "test-secret!"), + ("kid", "===="), + ] { + let request = RotateAcmeCredentialsRequest { + eab_kid: kid.into(), + eab_hmac_key: key.into(), + }; + let error = external_account_key(&request) + .err() + .expect("invalid EAB accepted"); + assert!(!format!("{error:#}").contains("test-secret")); + } + } +} diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs index 60d9e6912..cf8ff9e30 100644 --- a/dstack/gateway/src/distributed_certbot.rs +++ b/dstack/gateway/src/distributed_certbot.rs @@ -301,16 +301,22 @@ impl DistributedCertBot { /// /// Runs under the shared ACME lock, so a concurrent reconciliation or /// first-use registration on any node is refused for the duration. - pub async fn rotate_acme_credentials(&self) -> Result { + pub async fn rotate_acme_credentials( + &self, + external_account: Option<&certbot::ExternalAccountKey>, + ) -> Result { let rotation_lock = self.acquire_acme_lock("rotate ACME credentials")?; - let result = self.do_rotate_acme_credentials().await; + let result = self.do_rotate_acme_credentials(external_account).await; if let Err(err) = self.release_rotation_lock(&rotation_lock) { error!("failed to release ACME rotation lock: {err:?}"); } result } - async fn do_rotate_acme_credentials(&self) -> Result { + async fn do_rotate_acme_credentials( + &self, + external_account: Option<&certbot::ExternalAccountKey>, + ) -> Result { let configs = self.kv_store.list_zt_domain_configs(); let certbot_config = self.config()?; let acme_url = if certbot_config.acme_url.is_empty() { @@ -338,7 +344,7 @@ impl DistributedCertBot { // with no ZT domain yet registers here and gets an account URI, which is // what a `dns-persist-01` record has to name before the domain it // authorizes can ever be issued. - let account = AcmeClient::register_account(acme_url) + let account = AcmeClient::register_account_with_eab(acme_url, external_account) .await .context("failed to create replacement ACME account")?; let AcmeAccount { @@ -1384,7 +1390,7 @@ mod tests { .try_acquire_rotation_lock() .expect("lock should be free"); let err = certbot - .rotate_acme_credentials() + .rotate_acme_credentials(None) .await .expect_err("a concurrent run should be rejected"); assert!( @@ -1402,7 +1408,7 @@ mod tests { .try_acquire_rotation_lock(ROTATION_LOCK_TIMEOUT_SECS) .is_some()); let err = certbot - .rotate_acme_credentials() + .rotate_acme_credentials(None) .await .expect_err("rotation should be rejected while the KV lock is held"); assert!( @@ -1427,7 +1433,7 @@ mod tests { }) .expect("failed to store certbot config"); let err = certbot - .rotate_acme_credentials() + .rotate_acme_credentials(None) .await .expect_err("an unroutable ACME server cannot register an account"); assert!( diff --git a/dstack/gateway/src/main_service.rs b/dstack/gateway/src/main_service.rs index 1761fe84e..cce1c16b1 100644 --- a/dstack/gateway/src/main_service.rs +++ b/dstack/gateway/src/main_service.rs @@ -782,8 +782,9 @@ impl Proxy { pub(crate) async fn rotate_acme_credentials( &self, + external_account: Option<&certbot::ExternalAccountKey>, ) -> Result { - self.certbot.rotate_acme_credentials().await + self.certbot.rotate_acme_credentials(external_account).await } /// Get ACME info for all managed domains (or a specific domain)