Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ tokio = { version = ">=1.54.1, <2", features = ["full"] }
tokio-util = { version = ">=0.7.20, <0.8", features = ["rt"] }

# The floor moves with `release.helm.library` in .hyperi-ci.yaml.
scalo = { version = ">=2.14.32, <3", features = ["config-schema", "transport-kafka", "transport-grpc", "dlq-kafka", "config", "config-reload", "deployment", "version-check", "scaling", "cli-service", "top", "logger", "metrics", "service-metrics", "expression", "memory", "worker-batch", "governor", "secrets", "sink-stack", "geoip-download"] }
scalo = { version = ">=2.14.33, <3", features = ["config-schema", "transport-kafka", "transport-grpc", "dlq-kafka", "config", "config-reload", "deployment", "version-check", "scaling", "cli-service", "top", "logger", "metrics", "service-metrics", "expression", "memory", "worker-batch", "governor", "secrets", "sink-stack", "geoip-download"] }

# Steps scalo's jittered retry schedule (sink_stack backoff). Range MUST track
# scalo's: BackoffBuilder only applies to the ExponentialBuilder scalo returns.
Expand Down Expand Up @@ -105,7 +105,7 @@ test-case = ">=3.4.1, <4"
reqwest = { version = ">=0.12, <0.13", default-features = false, features = ["blocking", "rustls-tls-native-roots"] }

# The Kafka tests' broker image pin and TLS/SASL broker fixture.
scalo = { version = ">=2.14.32, <3", features = ["transport-kafka-test-support"] }
scalo = { version = ">=2.14.33, <3", features = ["transport-kafka-test-support"] }

# The status code a raw Push is refused with, through scalo's generated gRPC
# client. Same range as the copy scalo resolves, so cargo unifies to one.
Expand Down
20 changes: 3 additions & 17 deletions src/config/loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
//! 3. Figment env vars, `__` nesting, either separator after the prefix
//! (`DFE_LOADER__KAFKA__BROKERS` or `DFE_LOADER_KAFKA__BROKERS`)
//! 4. `./.env` in the working directory, read by scalo's cascade at startup
//! 5. Config file specified by --config or `DFE_LOADER_CONFIG`
//! 5. Config file specified by --config or `DFE_LOADER_CONFIG`, resolved by scalo's `run_app`
//! 6. Hard-coded defaults

use std::path::Path;
Expand Down Expand Up @@ -556,15 +556,14 @@ impl Config {
/// (chart / deployment-contract form) or `DFE_LOADER_KAFKA__SASL__USERNAME`
/// 4. `.env` file: the binary's `load_config` seeds scalo's cascade first,
/// which reads `./.env` and no parent directory's
/// 5. Config file (YAML, specified by `--config` or auto-detected)
/// 5. Config file (YAML): `config_path` is what scalo's `run_app` resolved from
/// `--config` or `DFE_LOADER_CONFIG`, else auto-detected
/// 6. Hard-coded defaults
pub fn load(config_path: Option<&str>) -> Result<Self> {
// 1. Start with hard-coded defaults
let mut config = Config::default();

// 2. Load YAML config file (overrides defaults).
let resolved = Self::resolve_config_path(config_path);
let config_path = resolved.as_deref();
if let Some(path) = config_path {
// A named file must be read: running on defaults instead hides a
// missing mount, and a SIGHUP reload would swap them in live.
Expand Down Expand Up @@ -595,19 +594,6 @@ impl Config {
Ok(config)
}

/// Resolve the config file: `--config`, else `DFE_LOADER_CONFIG`.
///
/// The env var is step 5 of this module's cascade and nothing read it --
/// scalo's `--config` arg carries no `env =`. The watcher resolves here too.
#[must_use]
pub fn resolve_config_path(cli_path: Option<&str>) -> Option<String> {
cli_path.map(String::from).or_else(|| {
std::env::var(format!("{ENV_PREFIX}_CONFIG"))
.ok()
.filter(|p| !p.is_empty())
})
}

/// Register all config sections in the global config registry.
///
/// Enables `/config` endpoint dump (with redaction) and change notifications.
Expand Down
6 changes: 3 additions & 3 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,9 @@ impl ServiceApp for App {
config: Self::Config,
mut runtime: ServiceRuntime,
) -> impl std::future::Future<Output = Result<(), CliError>> + Send {
// Same resolution Config::load() uses, so a config file named by
// DFE_LOADER_CONFIG is hot-reloaded like one named by --config.
let config_path = Config::resolve_config_path(self.common_args().config.as_deref());
// The file run_app loaded, so one named by DFE_LOADER_CONFIG is
// hot-reloaded like one named by --config.
let config_path = self.common_args().config_path(self.env_prefix());

async move {
let mut config = config;
Expand Down
158 changes: 137 additions & 21 deletions src/pipeline/orchestrator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1731,6 +1731,12 @@ impl Orchestrator {
self.count_placed(&dead_letters);
self.dlq_placed.extend(placed_offsets(&dead_letters));
}
Placement::Refused { dropped, placed } => {
self.drop_broker_refused(&dropped);
self.count_placed(&placed);
self.dlq_placed.extend(placed_offsets(&dropped));
self.dlq_placed.extend(placed_offsets(&placed));
}
Placement::Closed => {
warn!(
rows = dead_letters.len(),
Expand Down Expand Up @@ -2134,6 +2140,16 @@ impl Orchestrator {
}
}

/// Drop dead letters the DLQ's broker refused for their size once every
/// delivery report was in, so no retry can land them.
fn drop_broker_refused(&mut self, dropped: &[DlqEntry]) {
warn!(
rows = dropped.len(),
"The DLQ's broker refused dead letters for their size -- dropped, counted in pipeline_dead_letters_dropped_total and dfe_loader_rows_lost_total"
);
self.drop_dead_letters(dropped, DROPPED_TOO_LARGE);
}

/// Drop every dead letter read from a `dlq_load` topic, and return the rest.
///
/// Dead-lettering one again would feed it back to the topic it was read
Expand Down Expand Up @@ -2257,6 +2273,11 @@ impl Orchestrator {
self.count_placed(&dead_letters);
self.settle_dead_letters_as(&dead_letters, DeliveryStatus::Rejected);
}
Placement::Refused { dropped, placed } => {
self.drop_broker_refused(&dropped);
self.count_placed(&placed);
self.settle_dead_letters_as(&placed, DeliveryStatus::Rejected);
}
Placement::Closed => {
warn!(
rows = dead_letters.len(),
Expand Down Expand Up @@ -2817,6 +2838,10 @@ const DROPPED_NO_PAYLOAD: &str = "no_payload";
/// `reason` for dead letters read from a `dlq_load` topic that failed to load.
const DROPPED_DLQ_LOAD: &str = "dlq_load";

/// The `pipeline_dead_letters_dropped_total` reason for a dead letter the DLQ's
/// broker refused for its size, as scalo's pipeline labels it.
const DROPPED_TOO_LARGE: &str = "too_large";

/// The record sequence number a gRPC offset carries: the adapter stores each
/// token's `seq` as the message offset.
fn seq_of(offset: i64) -> u64 {
Expand Down Expand Up @@ -3153,10 +3178,16 @@ fn queue_rejected<'a>(
}

/// How handing a set of dead letters to the DLQ ended.
#[derive(Debug, PartialEq, Eq)]
#[derive(Debug)]
enum Placement {
/// A backend holds every one.
Written,
/// The Kafka-only DLQ's broker holds `placed` and refused `dropped` for
/// their size: no retry can land those, so they are dropped.
Refused {
dropped: Vec<DlqEntry>,
placed: Vec<DlqEntry>,
},
/// The DLQ's drain has exited, so no retry can land them.
Closed,
/// The DLQ still refused when the retry's expiry passed, or at shutdown.
Expand Down Expand Up @@ -3209,6 +3240,16 @@ async fn place_dead_letters(
return Placement::Written;
}
Ok(Err(scalo::dlq::DlqError::Closed)) => return Placement::Closed,
Ok(Err(scalo::dlq::DlqError::Refused(refused))) => {
let (dropped, placed) = split_refused(dead_letters, |entry| {
refused.contains(&dlq.cap(entry.clone()))
});
return if dropped.is_empty() {
Placement::Written
} else {
Placement::Refused { dropped, placed }
};
}
Ok(Err(e)) => e.to_string(),
Err(_elapsed) => {
return Placement::GaveUp(
Expand All @@ -3234,6 +3275,17 @@ async fn place_dead_letters(
}
}

/// Split `dead_letters` into those `is_refused` names and the rest.
fn split_refused(
dead_letters: &[DlqEntry],
is_refused: impl Fn(&DlqEntry) -> bool,
) -> (Vec<DlqEntry>, Vec<DlqEntry>) {
dead_letters
.iter()
.cloned()
.partition(|entry| is_refused(entry))
}

/// The `stage` label a DLQ write retry is counted under, as scalo's pipeline
/// names it.
const DLQ_STAGE: &str = "dlq";
Expand Down Expand Up @@ -4024,9 +4076,11 @@ mod tests {
let dead_letters =
rejected_entries(&payloads, &rejected_rows(rows, "server error code 117"));

assert_eq!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Written,
assert!(
matches!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Written
),
"a working DLQ takes every rejected row"
);

Expand Down Expand Up @@ -4345,9 +4399,11 @@ mod tests {

let payloads: Vec<Arc<[u8]>> = vec![Arc::from(&b"{\"n\":1}"[..])];
let dead_letters = rejected_entries(&payloads, &rejected_rows(1, "code 117"));
assert_eq!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Written,
assert!(
matches!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Written
),
"the gRPC DLQ refused a dead letter"
);

Expand Down Expand Up @@ -4425,9 +4481,11 @@ mod tests {
(source.topic.as_deref(), source.partition, source.offset),
(Some("dfe-events"), Some(3), Some(77))
);
assert_eq!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Written,
assert!(
matches!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Written
),
"a working DLQ takes the row"
);

Expand All @@ -4454,9 +4512,11 @@ mod tests {
row_json: Some(br#"{"_raw":"{\"user\":\"kaz\"}"}"#.to_vec()),
}];

assert_eq!(
place(&dlq, &rejected_entries(&payloads, &failed), in_30s()).await,
Placement::Written,
assert!(
matches!(
place(&dlq, &rejected_entries(&payloads, &failed), in_30s()).await,
Placement::Written
),
"a working DLQ takes the row"
);

Expand Down Expand Up @@ -4541,9 +4601,11 @@ mod tests {
"clickhouse_permanent_reject",
b"{\"kept\":1}".to_vec(),
)];
assert_eq!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Written,
assert!(
matches!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Written
),
"a working DLQ takes rows written after an earlier loss"
);
assert!(
Expand All @@ -4569,10 +4631,10 @@ mod tests {
})
.collect();

assert_eq!(
assert!(matches!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Written
);
));
assert_eq!(spooled_lines(&dir).len(), 3);
let _ = std::fs::remove_dir_all(&dir);
}
Expand All @@ -4589,9 +4651,11 @@ mod tests {
)];

let started = std::time::Instant::now();
assert_eq!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Closed,
assert!(
matches!(
place(&dlq, &dead_letters, in_30s()).await,
Placement::Closed
),
"a stopped DLQ was taken as a home for the rows"
);
assert!(
Expand Down Expand Up @@ -4710,6 +4774,58 @@ mod tests {
dlq.shutdown().await.expect("stop the DLQ");
}

/// `RefusedEntries` cannot be built outside scalo, so the broker's answer is
/// stood in for by the predicate `split_refused` takes.
#[test]
fn the_entries_a_refusal_names_split_from_the_rest() {
let entries: Vec<DlqEntry> = (1..=4).map(|seq| dead_letter_for(seq, 8)).collect();
let named = |entry: &DlqEntry| {
entry
.source
.as_ref()
.and_then(|s| s.offset)
.is_some_and(|offset| offset % 2 == 0)
};

let (dropped, placed) = split_refused(&entries, named);
assert_eq!(dropped.len(), 2);
assert_eq!(placed.len(), 2);
assert!(dropped.iter().all(named) && !placed.iter().any(named));

let (dropped, placed) = split_refused(&entries, |_| false);
assert!(dropped.is_empty() && placed.len() == 4);
}

#[test]
fn a_dead_letter_the_broker_refused_is_counted_lost_and_answered_dropped() {
for holds_answers in [false, true] {
let mut orchestrator = Orchestrator::new(Config::default());
orchestrator.holds_answers = holds_answers;
orchestrator.acks.admit(41);
let recorder = crate::metrics::counting::CountingRecorder::default();
let _local = metrics::set_default_local_recorder(&recorder);

orchestrator.drop_broker_refused(&[dead_letter_for(41, 8)]);

assert_eq!(
recorder.labelled(
crate::metrics::DEAD_LETTERS_DROPPED_TOTAL,
"reason",
"too_large"
),
1,
"the refused row was not counted (holds_answers {holds_answers})"
);
assert_eq!(recorder.total(crate::metrics::RETRIES_TOTAL), 0);
let released = orchestrator.acks.take_settled();
if holds_answers {
assert_eq!(released, vec![(DeliveryStatus::Dropped, vec![41])]);
} else {
assert_eq!(released.len(), 0);
}
}
}

/// A dead letter of the record read from `topic` at `offset`.
fn dead_letter_on(topic: &str, offset: i64) -> DlqEntry {
DlqEntry::new(
Expand Down
Loading
Loading