From 4ab964de1ab9987fcbb03e771697ab2cc7d7a67f Mon Sep 17 00:00:00 2001 From: Derek Date: Sat, 21 Nov 2026 14:52:15 +1100 Subject: [PATCH] fix(clickhouse): log why connections fail DNS failures, refused connections and untrusted certificates all logged "network error: client error (Connect)". The log gave no way to tell which one to fix. The client keeps the transport's reason only as the error's source. error_text appends each cause. The connector names its endpoint. - connecting to 127.0.0.1:8123: tcp connect error: Connection refused (os error 111) - connecting to clickhouse.invalid:8123: dns error: failed to lookup address information - connecting to 127.0.0.1:8443: invalid peer certificate: UnknownIssuer The tests use a closed loopback port, a .invalid name and a TLS server with a self-signed certificate. Add tower-service as a direct dependency. The lockfile had it through hyper-util. --- Cargo.lock | 1 + Cargo.toml | 3 + src/clickhouse/client_http.rs | 174 ++++++++++++++++++++++++++++++++++ src/clickhouse/error.rs | 55 ++++++++++- 4 files changed, 231 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 124d194..2054efb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1241,6 +1241,7 @@ dependencies = [ "tokio-test", "tokio-util", "tonic", + "tower-service", "tracing", "tracing-subscriber", "uuid", diff --git a/Cargo.toml b/Cargo.toml index 2922fa8..7612392 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,6 +74,9 @@ time = ">=0.3.55, <0.4" hyper-util = { version = ">=0.1.21, <0.2", features = ["client-legacy", "http1", "tokio"] } hyper-rustls = { version = ">=0.27.10, <0.28", default-features = false, features = ["http1", "tls12", "native-tokio", "aws-lc-rs"] } rustls = { version = ">=0.23.46, <0.24", default-features = false, features = ["std", "tls12", "aws_lc_rs"] } +# The connector trait http_transport's endpoint-naming wrapper implements. Same +# range as the copy hyper-util resolves, so cargo unifies to one. +tower-service = ">=0.3.3, <0.4" uuid = { version = ">=1.29.2, <2", features = ["v4", "v7"] } rustc-hash = ">=2.1.3, <3" diff --git a/src/clickhouse/client_http.rs b/src/clickhouse/client_http.rs index d95b71e..d597cdf 100644 --- a/src/clickhouse/client_http.rs +++ b/src/clickhouse/client_http.rs @@ -22,9 +22,12 @@ //! - Table existence checks //! - Health checks (ping) +use std::fmt; use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; +use std::task::{Context, Poll}; use std::time::Duration; use dfe_clickhouse::TcpClient; @@ -36,6 +39,7 @@ use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::rt::{TokioExecutor, TokioTimer}; use rustc_hash::FxHashMap; use serde::Deserialize; +use tower_service::Service; use super::config::{ClickHouseConfig, Timeouts}; use super::error::ClickHouseError; @@ -224,6 +228,78 @@ pub(crate) fn insert_sink( .with_timeouts(Some(timeouts.request), Some(timeouts.insert)) } +type BoxError = Box; + +/// A failed connect for the endpoint it was for. +/// +/// The connector's own error says what failed -- a lookup, a refusal, a +/// certificate -- but not for which host, and the client's text for it is +/// `client error (Connect)`. This sits above the connector's error in the +/// source chain, so every log line built from [`error_text`] names the host. +/// +/// [`error_text`]: super::error::error_text +#[derive(Debug)] +struct ConnectFailed { + endpoint: Arc, + source: BoxError, +} + +impl fmt::Display for ConnectFailed { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "connecting to {}", self.endpoint) + } +} + +impl std::error::Error for ConnectFailed { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + +/// A connector that fails with [`ConnectFailed`] for the endpoint it was built for. +#[derive(Clone)] +struct AtEndpoint { + endpoint: Arc, + inner: C, +} + +impl Service for AtEndpoint +where + C: Service, + C::Response: Send, + C::Future: Send + 'static, + C::Error: Into, +{ + type Response = C::Response; + type Error = BoxError; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + let endpoint = &self.endpoint; + self.inner.poll_ready(cx).map_err(|source| { + ConnectFailed { + endpoint: Arc::clone(endpoint), + source: source.into(), + } + .into() + }) + } + + fn call(&mut self, request: R) -> Self::Future { + let connecting = self.inner.call(request); + let endpoint = Arc::clone(&self.endpoint); + Box::pin(async move { + connecting.await.map_err(|source| { + ConnectFailed { + endpoint, + source: source.into(), + } + .into() + }) + }) + } +} + /// The HTTP client a `clickhouse::Client` sends through. /// /// HTTPS verifies the server's certificate and host name against the system @@ -232,11 +308,16 @@ fn http_transport(config: &ClickHouseConfig) -> Result { let mut connector = HttpConnector::new(); connector.set_keepalive(Some(TCP_KEEPALIVE)); connector.set_connect_timeout(Some(config.connect_timeout())); + let endpoint: Arc = config.primary_endpoint().unwrap_or_default().into(); let mut pool = HyperClient::builder(TokioExecutor::new()); // hyper-util evicts idle connections only when the pool has a timer. pool.pool_timer(TokioTimer::new()) .pool_idle_timeout(POOL_IDLE_TIMEOUT); if !config.tls { + let connector = AtEndpoint { + endpoint, + inner: connector, + }; return Ok(clickhouse::Client::with_http_client(pool.build(connector))); } connector.enforce_http(false); @@ -250,6 +331,10 @@ fn http_transport(config: &ClickHouseConfig) -> Result { .https_only() .enable_http1() .wrap_connector(connector); + let https = AtEndpoint { + endpoint, + inner: https, + }; Ok(clickhouse::Client::with_http_client(pool.build(https))) } @@ -1460,4 +1545,93 @@ mod tests { "the query gave up after {elapsed:?}, not at its 300 ms bound" ); } + + /// The text a health check against `config` fails with, bounded so a + /// resolver that never answers fails the test and does not hang it. + async fn health_check_failure(config: &ClickHouseConfig) -> String { + let client = ClickHouseQueryClient::new(config).unwrap(); + tokio::time::timeout(Duration::from_secs(20), client.health_check()) + .await + .expect("the health check ran past 20 s") + .expect_err("the health check reported success") + .to_string() + } + + /// A TLS server whose certificate no trust store holds, so a verifying + /// client refuses it mid-handshake. + fn self_signed_tls_server() -> SocketAddr { + let certified = rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_string()]).unwrap(); + let key = + rustls::pki_types::PrivatePkcs8KeyDer::from(certified.signing_key.serialize_der()); + let tls = Arc::new( + rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::aws_lc_rs::default_provider(), + )) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![certified.cert.der().clone()], key.into()) + .unwrap(), + ); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let (mut tcp, _) = listener.accept().unwrap(); + let mut conn = rustls::ServerConnection::new(tls).unwrap(); + // The client aborts on the certificate, so this ends in an error. + while conn.is_handshaking() && conn.complete_io(&mut tcp).is_ok() {} + }); + addr + } + + #[tokio::test] + async fn a_refused_connection_names_the_address_it_was_refused_at() { + let closed = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = closed.local_addr().unwrap(); + drop(closed); + + let text = health_check_failure(&http_config(addr, 2_000, 2_000)).await; + + assert!(text.contains(&format!("connecting to {addr}")), "{text}"); + assert!(text.contains("Connection refused"), "{text}"); + assert!(!text.contains("dns error"), "{text}"); + assert!(!text.contains("certificate"), "{text}"); + } + + #[tokio::test] + async fn a_name_that_does_not_resolve_is_reported_as_a_lookup_failure() { + // `.invalid` never resolves (RFC 6761), with or without network. + let config = ClickHouseConfig { + hosts: vec!["clickhouse.invalid:8123".to_string()], + transport: super::super::config::Transport::Http, + database: "default".to_string(), + connect_timeout_ms: 2_000, + request_timeout_ms: 2_000, + ..Default::default() + }; + + let text = health_check_failure(&config).await; + + assert!( + text.contains("connecting to clickhouse.invalid:8123"), + "{text}" + ); + assert!(text.contains("dns error"), "{text}"); + assert!(!text.contains("Connection refused"), "{text}"); + assert!(!text.contains("certificate"), "{text}"); + } + + #[tokio::test] + async fn an_untrusted_certificate_is_reported_with_the_verify_error() { + let addr = self_signed_tls_server(); + let mut config = http_config(addr, 2_000, 2_000); + config.tls = true; + + let text = health_check_failure(&config).await; + + assert!(text.contains(&format!("connecting to {addr}")), "{text}"); + assert!(text.contains("invalid peer certificate"), "{text}"); + assert!(!text.contains("dns error"), "{text}"); + assert!(!text.contains("Connection refused"), "{text}"); + } } diff --git a/src/clickhouse/error.rs b/src/clickhouse/error.rs index 67981d0..830c6d0 100644 --- a/src/clickhouse/error.rs +++ b/src/clickhouse/error.rs @@ -346,10 +346,30 @@ const ERROR_TEXT_CAP: usize = 1 << 20; pub fn error_text(err: &clickhouse::error::Error) -> String { capped(match ServerException::parse(err) { Some(exception) => server_text(exception.code, &exception.message), - None => err.to_string(), + None => with_causes(err), }) } +/// `err` and each cause beneath it, joined by `: `. The client wraps a failed +/// connect as `network error: client error (Connect)` and keeps what the +/// transport said -- a failed lookup, a refusal, a certificate fault -- only +/// as the error's source, so `to_string()` alone drops it. +fn with_causes(err: &(dyn std::error::Error + 'static)) -> String { + let mut text = err.to_string(); + let mut cause = err.source(); + while let Some(next) = cause { + let part = next.to_string(); + // An error that spells its source into its own message (`{0}` beside + // `#[source]`) must not print it twice. + if !text.contains(&part) { + text.push_str(": "); + text.push_str(&part); + } + cause = next.source(); + } + text +} + /// [`error_text`] for an error off the native protocol, which carries the /// server's code structurally. #[must_use] @@ -359,7 +379,7 @@ pub fn native_error_text(err: &dfe_clickhouse::Error) -> String { dfe_clickhouse::Error::BadResponse(body) => { return error_text(&clickhouse::error::Error::BadResponse(body.clone())); } - other => other.to_string(), + other => with_causes(other), }) } @@ -447,6 +467,37 @@ pub fn classify_from_message(msg: &str) -> ErrorCategory { mod tests { use super::*; + #[test] + fn error_text_keeps_a_cause_the_message_leaves_out_and_prints_one_it_repeats() { + // `Network` spells its source into its own message, so the chain + // must not print it twice. + let repeated = clickhouse::error::Error::Network("socket closed".into()); + assert_eq!(error_text(&repeated), "network error: socket closed"); + + let layered = clickhouse::error::Error::Network(Box::new(std::io::Error::other("outer"))); + assert_eq!(error_text(&layered), "network error: outer"); + + #[derive(Debug)] + struct Wrapper(std::io::Error); + impl std::fmt::Display for Wrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("client error (Connect)") + } + } + impl std::error::Error for Wrapper { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } + } + let hidden = clickhouse::error::Error::Network(Box::new(Wrapper(std::io::Error::other( + "tcp connect error", + )))); + assert_eq!( + error_text(&hidden), + "network error: client error (Connect): tcp connect error" + ); + } + #[test] fn test_connection_error_is_transient() { let err = ClickHouseError::Connection("network timeout".into());