diff --git a/crates/cli/src/commands/admin/bucket.rs b/crates/cli/src/commands/admin/bucket.rs index c5a363a..d5688c3 100644 --- a/crates/cli/src/commands/admin/bucket.rs +++ b/crates/cli/src/commands/admin/bucket.rs @@ -216,7 +216,7 @@ pub struct SetArgs { /// Source vendor family; drives addressing defaults #[arg(long, value_enum)] pub provider: ProviderArg, - /// Source endpoint as scheme://host[:port]; derived from --region for aws + /// Source endpoint as scheme://host\[:port\]; derived from --region for aws #[arg(long, value_name = "URL")] pub endpoint: Option, /// Source signing region; 'auto' is accepted for r2, minio and rustfs diff --git a/crates/core/src/lifecycle.rs b/crates/core/src/lifecycle.rs index 4407805..e4284cc 100644 --- a/crates/core/src/lifecycle.rs +++ b/crates/core/src/lifecycle.rs @@ -13,6 +13,7 @@ use serde::{Deserialize, Deserializer, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LifecycleConfiguration { /// Lifecycle rules + #[serde(alias = "Rules")] pub rules: Vec, } @@ -91,7 +92,7 @@ pub struct LifecycleRule { } #[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] struct LifecycleRuleInput { #[serde(alias = "ID")] id: String, @@ -109,6 +110,9 @@ struct LifecycleRuleInput { object_size_greater_than: Option, #[serde(default, alias = "ObjectSizeLessThan", alias = "object_size_less_than")] object_size_less_than: Option, + /// S3-shaped nested filter (`Filter.Prefix` / `Filter.Tag` / `Filter.And`). + #[serde(default, alias = "Filter")] + filter: Option, #[serde(default, alias = "Expiration")] expiration: Option, #[serde(default, alias = "Transition")] @@ -121,14 +125,27 @@ struct LifecycleRuleInput { noncurrent_version_transition: Option, #[serde(default, alias = "NoncurrentVersionTransitions")] noncurrent_version_transitions: Vec, - #[serde(default, alias = "AbortIncompleteMultipartUploadDays")] + #[serde( + default, + alias = "AbortIncompleteMultipartUploadDays", + alias = "abort_incomplete_multipart_upload_days" + )] abort_incomplete_multipart_upload_days: Option, + /// S3-shaped nested abort action (`AbortIncompleteMultipartUpload.DaysAfterInitiation`). + #[serde( + default, + alias = "AbortIncompleteMultipartUpload", + alias = "abort_incomplete_multipart_upload" + )] + abort_incomplete_multipart_upload: Option, + #[serde( default, alias = "ExpiredObjectDeleteMarker", alias = "expired_object_delete_marker" )] expired_object_delete_marker: Option, + #[serde( default, alias = "DelMarkerExpiration", @@ -137,8 +154,71 @@ struct LifecycleRuleInput { del_marker_expiration: Option, } +/// S3 `Filter` predicate: `Prefix`, a single `Tag`, an `And` combination, or a +/// standalone object-size bound. #[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct LifecycleFilterInput { + #[serde(default, alias = "Prefix")] + prefix: Option, + #[serde(default, alias = "Tag")] + tag: Option, + #[serde(default, alias = "And")] + and: Option, + #[serde( + default, + alias = "ObjectSizeGreaterThan", + alias = "object_size_greater_than" + )] + object_size_greater_than: Option, + #[serde(default, alias = "ObjectSizeLessThan", alias = "object_size_less_than")] + object_size_less_than: Option, +} + +/// S3 `Filter.And` combination of prefix, tags, and object-size bounds. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct LifecycleFilterAndInput { + #[serde(default, alias = "Prefix")] + prefix: Option, + #[serde(default, alias = "Tag")] + tag: Option, + #[serde(default, alias = "Tags")] + tags: Option>, + #[serde( + default, + alias = "ObjectSizeGreaterThan", + alias = "object_size_greater_than" + )] + object_size_greater_than: Option, + #[serde(default, alias = "ObjectSizeLessThan", alias = "object_size_less_than")] + object_size_less_than: Option, +} + +/// A single S3 tag predicate. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct LifecycleTagInput { + #[serde(alias = "Key")] + key: String, + #[serde(alias = "Value")] + value: String, +} + +/// S3-shaped `AbortIncompleteMultipartUpload` action. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct AbortIncompleteMultipartUploadInput { + #[serde( + default, + alias = "DaysAfterInitiation", + alias = "days_after_initiation" + )] + days_after_initiation: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] struct LifecycleExpirationInput { #[serde(default, alias = "Days")] days: Option, @@ -156,6 +236,12 @@ struct LifecycleExpirationInput { alias = "del_marker_expiration" )] del_marker_expiration: Option, + #[serde( + default, + alias = "ExpiredObjectDeleteMarker", + alias = "expired_object_delete_marker" + )] + expired_object_delete_marker: Option, } #[derive(Debug, Deserialize)] @@ -183,6 +269,7 @@ impl<'de> Deserialize<'de> for LifecycleRule { tags, object_size_greater_than, object_size_less_than, + filter, expiration: expiration_input, transition, transitions: mut transitions_input, @@ -190,10 +277,48 @@ impl<'de> Deserialize<'de> for LifecycleRule { noncurrent_version_transition, noncurrent_version_transitions: mut noncurrent_version_transitions_input, abort_incomplete_multipart_upload_days, + abort_incomplete_multipart_upload, expired_object_delete_marker, del_marker_expiration: top_level_del_marker_input, } = LifecycleRuleInput::deserialize(deserializer)?; + let filter_fields = resolve_filter_fields(filter).map_err(D::Error::custom)?; + let prefix = merge_exclusive(prefix, filter_fields.prefix, "prefix", "Filter.Prefix") + .map_err(D::Error::custom)?; + let tags = merge_exclusive(tags, filter_fields.tags, "tags", "Filter tags") + .map_err(D::Error::custom)?; + let object_size_greater_than = merge_exclusive( + object_size_greater_than, + filter_fields.object_size_greater_than, + "objectSizeGreaterThan", + "Filter.ObjectSizeGreaterThan", + ) + .map_err(D::Error::custom)?; + let object_size_less_than = merge_exclusive( + object_size_less_than, + filter_fields.object_size_less_than, + "objectSizeLessThan", + "Filter.ObjectSizeLessThan", + ) + .map_err(D::Error::custom)?; + + let nested_expired_object_delete_marker = expiration_input + .as_ref() + .and_then(|expiration| expiration.expired_object_delete_marker); + let expired_object_delete_marker = merge_equal_optional( + expired_object_delete_marker, + nested_expired_object_delete_marker, + "expiredObjectDeleteMarker", + ) + .map_err(D::Error::custom)?; + + let abort_incomplete_multipart_upload_days = merge_equal_optional( + abort_incomplete_multipart_upload_days, + abort_incomplete_multipart_upload.and_then(|action| action.days_after_initiation), + "abortIncompleteMultipartUploadDays", + ) + .map_err(D::Error::custom)?; + let (expiration, nested_del_marker) = match expiration_input { Some(expiration) => { let nested_del_marker = expiration.del_marker_expiration; @@ -254,6 +379,109 @@ impl<'de> Deserialize<'de> for LifecycleRule { } } +/// Filter predicates resolved from an S3-shaped `Filter` object. +#[derive(Default)] +struct FilterFields { + prefix: Option, + tags: Option>, + object_size_greater_than: Option, + object_size_less_than: Option, +} + +/// Flatten an S3 `Filter` (one of `Prefix`, `Tag`, `And`, or a standalone +/// object-size bound) into rc's flat fields. +/// +/// An empty `Filter` object stays valid: the server documents it as "applies to +/// every object in the bucket". +fn resolve_filter_fields( + filter: Option, +) -> std::result::Result { + let Some(filter) = filter else { + return Ok(FilterFields::default()); + }; + let predicate_count = usize::from(filter.prefix.is_some()) + + usize::from(filter.tag.is_some()) + + usize::from(filter.and.is_some()) + + usize::from(filter.object_size_greater_than.is_some()) + + usize::from(filter.object_size_less_than.is_some()); + if predicate_count > 1 { + return Err( + "S3 Filter allows exactly one of Prefix, Tag, And, ObjectSizeGreaterThan, or ObjectSizeLessThan" + .to_string(), + ); + } + if let Some(and) = filter.and { + let mut tags = HashMap::new(); + if let Some(tag) = and.tag { + tags.insert(tag.key, tag.value); + } + if let Some(tag_list) = and.tags { + for tag in tag_list { + if tags.insert(tag.key.clone(), tag.value).is_some() { + let key = tag.key; + return Err(format!("duplicate tag key in Filter.And: {key}")); + } + } + } + return Ok(FilterFields { + prefix: and.prefix, + tags: if tags.is_empty() { None } else { Some(tags) }, + object_size_greater_than: and.object_size_greater_than, + object_size_less_than: and.object_size_less_than, + }); + } + if let Some(prefix) = filter.prefix { + return Ok(FilterFields { + prefix: Some(prefix), + ..FilterFields::default() + }); + } + if let Some(tag) = filter.tag { + return Ok(FilterFields { + tags: Some(HashMap::from([(tag.key, tag.value)])), + ..FilterFields::default() + }); + } + if filter.object_size_greater_than.is_some() || filter.object_size_less_than.is_some() { + return Ok(FilterFields { + object_size_greater_than: filter.object_size_greater_than, + object_size_less_than: filter.object_size_less_than, + ..FilterFields::default() + }); + } + Ok(FilterFields::default()) +} + +/// Combine a flat field with its nested equivalent; both set at once is ambiguous. +fn merge_exclusive( + flat: Option, + nested: Option, + flat_name: &str, + nested_name: &str, +) -> std::result::Result, String> { + match (flat, nested) { + (Some(_), Some(_)) => Err(format!( + "lifecycle rule sets both {flat_name} and {nested_name}; specify one" + )), + (value, None) | (None, value) => Ok(value), + } +} + +/// Combine two optional spellings of the same value; equal duplicates are fine. +fn merge_equal_optional( + primary: Option, + secondary: Option, + field_name: &str, +) -> std::result::Result, String> { + match (primary, secondary) { + (Some(primary_value), Some(secondary_value)) if primary_value != secondary_value => { + Err(format!("conflicting {field_name} values in lifecycle rule")) + } + (value, None) | (None, value) => Ok(value), + (Some(value), Some(_)) => Ok(Some(value)), + } +} + fn normalize_del_marker_input( input: Option, fallback_days: Option, diff --git a/crates/core/tests/lifecycle_import_dialect.rs b/crates/core/tests/lifecycle_import_dialect.rs new file mode 100644 index 0000000..9ea2a3a --- /dev/null +++ b/crates/core/tests/lifecycle_import_dialect.rs @@ -0,0 +1,157 @@ +//! `rc ilm rule import` JSON dialect tests. +//! +//! Import accepts the rc camelCase dialect plus the common alternative +//! spellings people paste (snake_case, S3/PascalCase, and the S3 JSON shape +//! with a nested `filter`/`abortIncompleteMultipartUpload`). Unknown rule +//! fields are rejected instead of silently dropped: a dropped action field +//! used to import a degraded rule that the server then refused with a +//! confusing "rule must have an action" error. + +use rc_core::lifecycle::LifecycleConfiguration; + +fn parse(json: &str) -> LifecycleConfiguration { + serde_json::from_str(json).expect("lifecycle import should parse") +} + +fn parse_error(json: &str) -> String { + serde_json::from_str::(json) + .expect_err("lifecycle import should fail") + .to_string() +} + +#[test] +fn camel_case_flat_dialect_round_trips() { + let config = parse( + r#"{"rules":[{"id":"r","status":"Enabled","prefix":"v1/","abortIncompleteMultipartUploadDays":1}]}"#, + ); + assert_eq!(config.rules[0].prefix.as_deref(), Some("v1/")); + assert_eq!( + config.rules[0].abort_incomplete_multipart_upload_days, + Some(1) + ); +} + +#[test] +fn snake_case_flat_dialect_is_accepted() { + let config = parse( + r#"{"rules":[{"id":"r","status":"Enabled","prefix":"v1/","abort_incomplete_multipart_upload_days":1}]}"#, + ); + assert_eq!(config.rules[0].prefix.as_deref(), Some("v1/")); + assert_eq!( + config.rules[0].abort_incomplete_multipart_upload_days, + Some(1) + ); +} + +#[test] +fn pascal_case_flat_dialect_is_accepted() { + let config = parse( + r#"{"Rules":[{"ID":"r","Status":"Enabled","Prefix":"v1/","AbortIncompleteMultipartUploadDays":1}]}"#, + ); + assert_eq!(config.rules[0].prefix.as_deref(), Some("v1/")); + assert_eq!( + config.rules[0].abort_incomplete_multipart_upload_days, + Some(1) + ); +} + +#[test] +fn s3_shaped_filter_and_abort_action_are_accepted() { + let config = parse( + r#"{"rules":[{"id":"r","status":"Enabled","filter":{"prefix":"v1/"},"abortIncompleteMultipartUpload":{"daysAfterInitiation":2}}]}"#, + ); + assert_eq!(config.rules[0].prefix.as_deref(), Some("v1/")); + assert_eq!( + config.rules[0].abort_incomplete_multipart_upload_days, + Some(2) + ); +} + +#[test] +fn s3_filter_and_combination_is_flattened() { + let config = parse( + r#"{"rules":[{"id":"r","status":"Enabled","filter":{"And":{"Prefix":"logs/","Tags":[{"Key":"env","Value":"prod"}],"ObjectSizeGreaterThan":100,"ObjectSizeLessThan":1000}}}]}"#, + ); + assert_eq!(config.rules[0].prefix.as_deref(), Some("logs/")); + assert_eq!( + config.rules[0] + .tags + .as_ref() + .and_then(|tags| tags.get("env")), + Some(&"prod".to_string()) + ); + assert_eq!(config.rules[0].object_size_greater_than, Some(100)); + assert_eq!(config.rules[0].object_size_less_than, Some(1000)); +} + +#[test] +fn s3_standalone_object_size_greater_than_filter_is_accepted() { + let config = parse( + r#"{"Rules":[{"ID":"size-filter","Status":"Enabled","Filter":{"ObjectSizeGreaterThan":1024},"Expiration":{"Days":30}}]}"#, + ); + assert_eq!(config.rules[0].object_size_greater_than, Some(1024)); + assert_eq!(config.rules[0].expiration.clone().unwrap().days, Some(30)); +} + +#[test] +fn s3_standalone_object_size_less_than_filter_is_accepted() { + let config = parse( + r#"{"rules":[{"id":"r","status":"Enabled","filter":{"ObjectSizeLessThan":65536},"expiration":{"days":7}}]}"#, + ); + assert_eq!(config.rules[0].object_size_less_than, Some(65536)); + assert_eq!(config.rules[0].expiration.clone().unwrap().days, Some(7)); +} + +#[test] +fn nested_expired_object_delete_marker_is_accepted() { + let config = parse( + r#"{"rules":[{"id":"r","status":"Enabled","expiration":{"ExpiredObjectDeleteMarker":true}}]}"#, + ); + assert_eq!(config.rules[0].expired_object_delete_marker, Some(true)); +} + +#[test] +fn unknown_rule_field_is_rejected_instead_of_dropped() { + let error = parse_error( + r#"{"rules":[{"id":"r","status":"Enabled","prefix":"v1/","daysAfterInitiation":1}]}"#, + ); + assert!( + error.contains("unknown field") && error.contains("daysAfterInitiation"), + "error should name the unknown field: {error}" + ); +} + +#[test] +fn flat_and_nested_abort_days_conflict_is_rejected() { + let error = parse_error( + r#"{"rules":[{"id":"r","status":"Enabled","abortIncompleteMultipartUploadDays":1,"abortIncompleteMultipartUpload":{"daysAfterInitiation":2}}]}"#, + ); + assert!( + error.contains("abortIncompleteMultipartUploadDays"), + "{error}" + ); +} + +#[test] +fn flat_prefix_and_filter_conflict_is_rejected() { + let error = parse_error( + r#"{"rules":[{"id":"r","status":"Enabled","prefix":"a/","filter":{"prefix":"b/"}}]}"#, + ); + assert!( + error.contains("prefix") && error.contains("Filter.Prefix"), + "{error}" + ); +} + +#[test] +fn filter_with_two_top_level_predicates_is_rejected() { + let error = parse_error( + r#"{"rules":[{"id":"r","status":"Enabled","filter":{"prefix":"a/","tag":{"key":"env","value":"prod"}}}]}"#, + ); + assert!( + error.contains( + "exactly one of Prefix, Tag, And, ObjectSizeGreaterThan, or ObjectSizeLessThan" + ), + "{error}" + ); +} diff --git a/crates/s3/src/lifecycle_xml.rs b/crates/s3/src/lifecycle_xml.rs index 7ad38e1..f2f3a3f 100644 --- a/crates/s3/src/lifecycle_xml.rs +++ b/crates/s3/src/lifecycle_xml.rs @@ -543,6 +543,36 @@ fn xml_escape(value: &str) -> String { mod tests { use super::*; + #[test] + fn abort_only_rule_roundtrips_prefix_and_abort_action() { + let rules = vec![super::LifecycleRule { + id: "abort-only".to_string(), + status: super::LifecycleRuleStatus::Enabled, + prefix: Some("v1/".to_string()), + tags: None, + object_size_greater_than: None, + object_size_less_than: None, + expiration: None, + del_marker_expiration: None, + transition: None, + transitions: Vec::new(), + noncurrent_version_expiration: None, + noncurrent_version_transition: None, + noncurrent_version_transitions: Vec::new(), + abort_incomplete_multipart_upload_days: Some(1), + expired_object_delete_marker: None, + }]; + let xml = super::build_lifecycle_configuration_xml(&rules); + assert!( + xml.contains("1"), + "abort action missing from XML: {xml}" + ); + assert!( + xml.contains("v1/"), + "prefix missing from XML: {xml}" + ); + } + #[test] fn lifecycle_xml_roundtrip_preserves_extension_fields_and_standard_actions() { let mut tags = HashMap::new();