diff --git a/crates/core/src/api/list.rs b/crates/core/src/api/list.rs index 1d333d3..47f1a03 100644 --- a/crates/core/src/api/list.rs +++ b/crates/core/src/api/list.rs @@ -387,7 +387,7 @@ mod tests { fn make_config(backend_prefix: Option<&str>) -> BucketConfig { BucketConfig { name: "test-bucket".to_string(), - backend_type: "s3".to_string(), + backend_type: crate::types::BackendType::S3, backend_prefix: backend_prefix.map(|s| s.to_string()), anonymous_access: false, allowed_roles: vec![], diff --git a/crates/core/src/auth/authorize.rs b/crates/core/src/auth/authorize.rs index 48d860b..1ce4a33 100644 --- a/crates/core/src/auth/authorize.rs +++ b/crates/core/src/auth/authorize.rs @@ -169,7 +169,7 @@ mod tests { fn bucket(name: &str, anonymous: bool) -> BucketConfig { BucketConfig { name: name.into(), - backend_type: "s3".into(), + backend_type: crate::types::BackendType::S3, backend_prefix: None, anonymous_access: anonymous, allowed_roles: vec![], diff --git a/crates/core/src/backend/mod.rs b/crates/core/src/backend/mod.rs index 99beeca..7209fa5 100644 --- a/crates/core/src/backend/mod.rs +++ b/crates/core/src/backend/mod.rs @@ -162,47 +162,34 @@ impl StoreBuilder { /// an HTTP connector), then call [`StoreBuilder::build`] or /// [`StoreBuilder::build_signer`]. pub fn create_builder(config: &BucketConfig) -> Result { - let backend_type = config.parsed_backend_type().ok_or_else(|| { - ProxyError::ConfigError(format!( - "unsupported backend_type: '{}'", - config.backend_type - )) - })?; - - match backend_type { - BackendType::S3 => { - let mut b = AmazonS3Builder::new(); + /// Apply every `backend_options` entry whose key the provider recognizes. + /// A macro rather than a fn because each builder has its own config-key type. + macro_rules! with_options { + ($builder:expr) => {{ + let mut b = $builder; for (k, v) in &config.backend_options { if let Ok(key) = k.parse() { b = b.with_config(key, v); } } - Ok(StoreBuilder::S3(b)) - } + b + }}; + } + + match config.backend_type { + BackendType::S3 => Ok(StoreBuilder::S3(with_options!(AmazonS3Builder::new()))), #[cfg(feature = "azure")] - BackendType::Azure => { - let mut b = MicrosoftAzureBuilder::new(); - for (k, v) in &config.backend_options { - if let Ok(key) = k.parse() { - b = b.with_config(key, v); - } - } - Ok(StoreBuilder::Azure(b)) - } + BackendType::Azure => Ok(StoreBuilder::Azure(with_options!( + MicrosoftAzureBuilder::new() + ))), #[cfg(not(feature = "azure"))] BackendType::Azure => Err(ProxyError::ConfigError( "Azure backend support not enabled (requires 'azure' feature)".into(), )), #[cfg(feature = "gcp")] - BackendType::Gcs => { - let mut b = GoogleCloudStorageBuilder::new(); - for (k, v) in &config.backend_options { - if let Ok(key) = k.parse() { - b = b.with_config(key, v); - } - } - Ok(StoreBuilder::Gcs(b)) - } + BackendType::Gcs => Ok(StoreBuilder::Gcs(with_options!( + GoogleCloudStorageBuilder::new() + ))), #[cfg(not(feature = "gcp"))] BackendType::Gcs => Err(ProxyError::ConfigError( "GCS backend support not enabled (requires 'gcp' feature)".into(), diff --git a/crates/core/src/backend/multipart.rs b/crates/core/src/backend/multipart.rs index f40ba09..07d4b19 100644 --- a/crates/core/src/backend/multipart.rs +++ b/crates/core/src/backend/multipart.rs @@ -157,7 +157,7 @@ mod tests { backend_options.insert("bucket_name".into(), "my-backend-bucket".into()); BucketConfig { name: "test".into(), - backend_type: "s3".into(), + backend_type: crate::types::BackendType::S3, backend_prefix: None, anonymous_access: false, allowed_roles: vec![], diff --git a/crates/core/src/backend/url_signer.rs b/crates/core/src/backend/url_signer.rs index bf98002..0274dc2 100644 --- a/crates/core/src/backend/url_signer.rs +++ b/crates/core/src/backend/url_signer.rs @@ -8,7 +8,7 @@ use super::create_builder; use crate::error::ProxyError; -use crate::types::BucketConfig; +use crate::types::{BackendType, BucketConfig}; use object_store::signer::Signer; use std::sync::Arc; @@ -39,10 +39,8 @@ struct UnsignedUrlSigner { impl UnsignedUrlSigner { fn from_config(config: &BucketConfig) -> Result { - use crate::types::BackendType; - - match config.parsed_backend_type() { - Some(BackendType::Azure) => { + match config.backend_type { + BackendType::Azure => { let account_name = config.option("account_name").unwrap_or(""); let container = config.option("container_name").unwrap_or(""); Ok(Self { @@ -50,15 +48,14 @@ impl UnsignedUrlSigner { bucket: container.to_string(), }) } - Some(BackendType::Gcs) => { + BackendType::Gcs => { let bucket = config.option("bucket_name").unwrap_or(""); Ok(Self { endpoint: "https://storage.googleapis.com".to_string(), bucket: bucket.to_string(), }) } - _ => { - // S3 or unknown — use endpoint + bucket_name + BackendType::S3 => { let endpoint = config .option("endpoint") .unwrap_or("https://s3.amazonaws.com"); diff --git a/crates/core/src/middleware.rs b/crates/core/src/middleware.rs index 00d68e6..a4d1bab 100644 --- a/crates/core/src/middleware.rs +++ b/crates/core/src/middleware.rs @@ -279,7 +279,7 @@ mod tests { static BUCKET_CONFIG: std::sync::LazyLock = std::sync::LazyLock::new(|| BucketConfig { name: "test".to_string(), - backend_type: "s3".to_string(), + backend_type: crate::types::BackendType::S3, backend_prefix: None, anonymous_access: false, allowed_roles: Vec::new(), diff --git a/crates/core/src/proxy/tests.rs b/crates/core/src/proxy/tests.rs index c8acb5d..ed82555 100644 --- a/crates/core/src/proxy/tests.rs +++ b/crates/core/src/proxy/tests.rs @@ -107,13 +107,13 @@ fn test_bucket_config(name: &str) -> BucketConfig { // A bucket named `azure-*` resolves to a non-S3 backend so tests can // exercise the non-S3 rejection paths; everything else is S3. let backend_type = if name.starts_with("azure") { - "azure" + crate::types::BackendType::Azure } else { - "s3" + crate::types::BackendType::S3 }; BucketConfig { name: name.to_string(), - backend_type: backend_type.into(), + backend_type, backend_prefix: None, anonymous_access: true, allowed_roles: vec![], @@ -1149,7 +1149,7 @@ fn same_s3_endpoint_matches_shared_endpoint_and_region() { assert!(!same_s3_endpoint(&a, &c)); // A non-S3 backend is never a copy-compatible store. - a.backend_type = "azure".into(); + a.backend_type = crate::types::BackendType::Azure; let d = test_bucket_config("dst3"); assert!(!same_s3_endpoint(&a, &d)); } diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 1c82624..33eba5a 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -1,9 +1,11 @@ //! Shared types used across the proxy. +use crate::error::ProxyError; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt; +use std::str::FromStr; /// Owner identity for S3 ListBuckets responses. #[derive(Debug, Clone, Serialize)] @@ -20,8 +22,10 @@ pub struct BucketConfig { /// The virtual bucket name exposed to clients. pub name: String, - /// Provider type: "s3", "az", "gcs", etc. - pub backend_type: String, + /// Backend provider. In config files this is `"s3"`, `"azure"` (alias + /// `"az"`), or `"gcs"` (alias `"gs"`); an unknown value is rejected when + /// the config is parsed rather than on the first request. + pub backend_type: BackendType, /// Optional prefix to prepend to all keys when forwarding. pub backend_prefix: Option, @@ -78,32 +82,60 @@ impl fmt::Debug for BucketConfig { } /// Known backend provider types. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// +/// Serializes as the canonical lowercase name (`s3`, `azure`, `gcs`). +/// Deserialization and [`FromStr`] also accept the short aliases `az` and `gs`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum BackendType { /// Amazon S3 or S3-compatible storage. S3, /// Azure Blob Storage. + #[serde(alias = "az")] Azure, /// Google Cloud Storage. + #[serde(alias = "gs")] Gcs, } -impl BucketConfig { - /// Parse the `backend_type` string into a known [`BackendType`]. - pub fn parsed_backend_type(&self) -> Option { - match self.backend_type.as_str() { - "s3" => Some(BackendType::S3), - "az" | "azure" => Some(BackendType::Azure), - "gcs" | "gs" => Some(BackendType::Gcs), - _ => None, +impl BackendType { + /// The canonical config-file spelling of this backend type. + pub fn as_str(self) -> &'static str { + match self { + Self::S3 => "s3", + Self::Azure => "azure", + Self::Gcs => "gcs", + } + } +} + +impl fmt::Display for BackendType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for BackendType { + type Err = ProxyError; + + fn from_str(s: &str) -> Result { + match s { + "s3" => Ok(Self::S3), + "az" | "azure" => Ok(Self::Azure), + "gcs" | "gs" => Ok(Self::Gcs), + other => Err(ProxyError::ConfigError(format!( + "unsupported backend_type: '{other}' (expected s3, azure, or gcs)" + ))), } } +} +impl BucketConfig { /// Whether this is an S3 backend. Operations that go through raw signed /// HTTP rather than presigned URLs — multipart uploads and batch delete — /// are gated on this. pub fn is_s3_backend(&self) -> bool { - matches!(self.parsed_backend_type(), Some(BackendType::S3)) + self.backend_type == BackendType::S3 } /// Look up a value in `backend_options`. @@ -531,6 +563,41 @@ impl S3Operation { mod tests { use super::*; + #[test] + fn backend_type_parses_canonical_names_and_aliases() { + assert_eq!("s3".parse::().unwrap(), BackendType::S3); + assert_eq!("azure".parse::().unwrap(), BackendType::Azure); + assert_eq!("az".parse::().unwrap(), BackendType::Azure); + assert_eq!("gcs".parse::().unwrap(), BackendType::Gcs); + assert_eq!("gs".parse::().unwrap(), BackendType::Gcs); + assert!(matches!( + "ftp".parse::(), + Err(ProxyError::ConfigError(_)) + )); + } + + #[test] + fn backend_type_serde_accepts_aliases_and_emits_canonical_name() { + let parsed: BackendType = serde_json::from_str("\"az\"").unwrap(); + assert_eq!(parsed, BackendType::Azure); + let parsed: BackendType = serde_json::from_str("\"gs\"").unwrap(); + assert_eq!(parsed, BackendType::Gcs); + assert_eq!( + serde_json::to_string(&BackendType::Azure).unwrap(), + "\"azure\"" + ); + assert!(serde_json::from_str::("\"ftp\"").is_err()); + } + + #[test] + fn bucket_config_rejects_unknown_backend_type_at_parse_time() { + let err = serde_json::from_str::( + r#"{"name":"b","backend_type":"ftp","anonymous_access":true}"#, + ) + .unwrap_err(); + assert!(err.to_string().contains("ftp"), "got: {err}"); + } + #[test] fn test_action() { let op = S3Operation::GetObject { @@ -599,7 +666,7 @@ mod tests { backend_options.insert("skip_signature".to_string(), "true".to_string()); BucketConfig { name: "acct:product".to_string(), - backend_type: "s3".to_string(), + backend_type: BackendType::S3, backend_prefix: None, anonymous_access: true, allowed_roles: vec![], diff --git a/crates/core/tests/conditional_writes.rs b/crates/core/tests/conditional_writes.rs index b02fba3..908370a 100644 --- a/crates/core/tests/conditional_writes.rs +++ b/crates/core/tests/conditional_writes.rs @@ -160,7 +160,7 @@ fn test_bucket_config(name: &str) -> BucketConfig { ); BucketConfig { name: name.to_string(), - backend_type: "s3".into(), + backend_type: multistore::types::BackendType::S3, backend_prefix: None, anonymous_access: true, allowed_roles: vec![], diff --git a/crates/core/tests/key_encoding_contract.rs b/crates/core/tests/key_encoding_contract.rs index e9779d9..8163f95 100644 --- a/crates/core/tests/key_encoding_contract.rs +++ b/crates/core/tests/key_encoding_contract.rs @@ -54,7 +54,7 @@ fn bucket_config(with_creds: bool) -> BucketConfig { } BucketConfig { name: "test".into(), - backend_type: "s3".into(), + backend_type: multistore::types::BackendType::S3, backend_prefix: None, anonymous_access: !with_creds, allowed_roles: vec![], diff --git a/crates/oidc-provider/src/backend_auth.rs b/crates/oidc-provider/src/backend_auth.rs index 387a1d7..77ab561 100644 --- a/crates/oidc-provider/src/backend_auth.rs +++ b/crates/oidc-provider/src/backend_auth.rs @@ -11,7 +11,7 @@ use multistore::error::ProxyError; use multistore::middleware::{DispatchContext, Middleware, Next}; use multistore::route_handler::HandlerAction; -use multistore::types::BucketConfig; +use multistore::types::{BackendType, BucketConfig}; use std::borrow::Cow; use std::collections::HashMap; @@ -82,8 +82,8 @@ impl AwsBackendAuth { if config.option("auth_type") != Some("oidc") { return Ok(None); } - match config.backend_type.as_str() { - "s3" => self.resolve_aws(config).await.map(Some), + match config.backend_type { + BackendType::S3 => self.resolve_aws(config).await.map(Some), other => Err(ProxyError::ConfigError(format!( "OIDC backend auth not yet supported for backend_type '{other}'" ))), @@ -99,8 +99,8 @@ impl Middleware for AwsBackendAuth { ) -> Result { if let Some(ref bucket_config) = ctx.bucket_config { if bucket_config.option("auth_type") == Some("oidc") { - match bucket_config.backend_type.as_str() { - "s3" => { + match bucket_config.backend_type { + BackendType::S3 => { let options = self.resolve_aws(bucket_config).await?; ctx.bucket_config = Some(Cow::Owned(BucketConfig { backend_options: options, @@ -228,7 +228,7 @@ mod tests { opts.insert("region".into(), "us-east-1".into()); BucketConfig { name: "test".into(), - backend_type: "s3".into(), + backend_type: BackendType::S3, backend_prefix: None, anonymous_access: false, allowed_roles: vec![], @@ -247,7 +247,7 @@ mod tests { opts.insert("bucket_name".into(), "my-bucket".into()); BucketConfig { name: "test".into(), - backend_type: "s3".into(), + backend_type: BackendType::S3, backend_prefix: None, anonymous_access: false, allowed_roles: vec![], diff --git a/crates/static-config/src/lib.rs b/crates/static-config/src/lib.rs index 4f71ce0..01e7eec 100644 --- a/crates/static-config/src/lib.rs +++ b/crates/static-config/src/lib.rs @@ -274,7 +274,7 @@ mod tests { owner_display_name: None, buckets: vec![BucketConfig { name: "my-bucket".into(), - backend_type: "s3".into(), + backend_type: multistore::types::BackendType::S3, backend_prefix: None, anonymous_access: true, allowed_roles: vec![], diff --git a/docs/configuration/buckets.md b/docs/configuration/buckets.md index db1c02f..3f46540 100644 --- a/docs/configuration/buckets.md +++ b/docs/configuration/buckets.md @@ -25,7 +25,7 @@ secret_access_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | Field | Type | Required | Description | |-------|------|----------|-------------| | `name` | string | Yes | Client-visible bucket name | -| `backend_type` | string | Yes | Backend provider: `"s3"`, `"az"`, or `"gcs"` | +| `backend_type` | string | Yes | Backend provider: `"s3"`, `"azure"` (alias `"az"`), or `"gcs"` (alias `"gs"`). Any other value is a config error. | | `backend_prefix` | string | No | Prefix prepended to keys when forwarding to the backend | | `anonymous_access` | bool | Yes | Allow GET/HEAD/LIST without authentication (must be set explicitly; omitting it is a config error) | | `allowed_roles` | string[] | No | Role IDs that can be assumed for this bucket | diff --git a/docs/extending/custom-backend.md b/docs/extending/custom-backend.md index 43744a2..297a683 100644 --- a/docs/extending/custom-backend.md +++ b/docs/extending/custom-backend.md @@ -138,7 +138,7 @@ async fn send_raw( The `backend` module provides shared helpers: -- **`create_builder(config)`** — Dispatches on `backend_type` ("s3", "az", "gcs"), iterates `backend_options` with `with_config()`, and returns a `StoreBuilder` that can be customized (e.g. inject an HTTP connector) before calling `.build()` or `.build_signer()` +- **`create_builder(config)`** — Dispatches on the `BackendType` enum (`S3`, `Azure`, `Gcs`), iterates `backend_options` with `with_config()`, and returns a `StoreBuilder` that can be customized (e.g. inject an HTTP connector) before calling `.build()` or `.build_signer()` - **`build_signer(config)`** — Returns the appropriate signer: `object_store`'s built-in signer for authenticated backends, or `UnsignedUrlSigner` for anonymous backends These handle the multi-provider dispatch logic so your backend implementation only needs to provide the HTTP transport layer. diff --git a/docs/reference/config-example.md b/docs/reference/config-example.md index f04a4c6..f6ef226 100644 --- a/docs/reference/config-example.md +++ b/docs/reference/config-example.md @@ -10,7 +10,7 @@ A complete, annotated configuration file showing all available options. # A publicly accessible S3 bucket (anonymous reads allowed) [[buckets]] name = "public-data" # Client-visible bucket name -backend_type = "s3" # Backend provider: "s3", "az", or "gcs" +backend_type = "s3" # Backend provider: "s3", "azure" (or "az"), "gcs" (or "gs") anonymous_access = true # Allow GET/HEAD/LIST without auth allowed_roles = [] # No STS roles (anonymous only)