From d1c8df4cac0098d84bad45fed62275fcfdf248e2 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Thu, 13 Aug 2026 18:35:52 +0200 Subject: [PATCH 01/10] refactor(config): extract build_config, add load_optional + Serialize --- src/services/config.rs | 221 +++++++++++++---------------- src/tests/services/config_tests.rs | 71 +++++++++ 2 files changed, 171 insertions(+), 121 deletions(-) diff --git a/src/services/config.rs b/src/services/config.rs index ddb56c1..9142c78 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] use crate::core::context::Context; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json; use std::collections::HashMap; use std::fs::File; @@ -12,7 +12,7 @@ use toml; use tracing::info; use uuid::Uuid; -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "lowercase")] pub enum DbType { Mysql, @@ -49,7 +49,7 @@ impl DbType { } #[allow(dead_code)] -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct DatabaseConfig { pub name: String, pub database: String, @@ -68,7 +68,7 @@ pub struct DatabaseConfig { } #[allow(dead_code)] -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct DatabasesConfig { pub databases: Vec, } @@ -98,6 +98,86 @@ pub struct InputDatabasesConfig { pub databases: Vec, } +fn required(opt: &Option, db_name: &str, field_name: &str) -> Result { + match opt { + Some(v) => Ok(v.clone()), + None => Err(format!( + "Missing required field '{}' for database '{}'", + field_name, db_name + )), + } +} + +fn optional(opt: &Option) -> T { + opt.clone().unwrap_or_default() +} + +pub fn build_config(db: InputDatabaseConfig) -> Result { + if Uuid::parse_str(&db.generated_id).is_err() { + return Err(format!("Invalid UUID for database '{}'", db.name)); + } + + let username = match db.db_type { + DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb + | DbType::Mssql => required(&db.username, &db.name, "username")?, + _ => optional(&db.username), + }; + let password = match db.db_type { + DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb + | DbType::Mssql => required(&db.password, &db.name, "password")?, + _ => optional(&db.password), + }; + let host = match db.db_type { + DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb + | DbType::MongoDB | DbType::Redis | DbType::Firebird | DbType::Valkey | DbType::Mssql => { + required(&db.host, &db.name, "host")? + } + DbType::Sqlite | DbType::DockerVolume => optional(&db.host), + }; + let port = match db.db_type { + DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb + | DbType::MongoDB | DbType::Redis | DbType::Firebird | DbType::Valkey | DbType::Mssql => { + required(&db.port, &db.name, "port")? + } + DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0), + }; + let database_name = match db.db_type { + DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => { + optional(&db.database) + } + DbType::PostgresqlCluster => db.database.clone().unwrap_or_else(|| "postgres".to_string()), + _ => required(&db.database, &db.name, "database")?, + }; + let path_val = match db.db_type { + DbType::Sqlite => required(&db.path, &db.name, "path")?, + _ => optional(&db.path), + }; + let max_packet_size = match db.db_type { + DbType::Mysql | DbType::Mariadb => db.max_packet_size.unwrap_or_else(|| "512M".to_string()), + _ => String::new(), + }; + let volume_name = match db.db_type { + DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?, + _ => optional(&db.volume_name), + }; + + Ok(DatabaseConfig { + name: db.name, + database: database_name, + db_type: db.db_type, + username, + password, + host, + port, + generated_id: db.generated_id, + path: path_val, + max_packet_size, + volume_name, + container_name: db.container_name.clone(), + options: db.options.unwrap_or_default(), + }) +} + pub struct ConfigService { ctx: Arc, } @@ -150,128 +230,27 @@ impl ConfigService { _ => return Err("Unsupported config file format. Use .json or .toml".to_string()), }; - fn required( - opt: &Option, - db_name: &str, - field_name: &str, - ) -> Result { - match opt { - Some(v) => Ok(v.clone()), - None => { - let msg = format!( - "Missing required field '{}' for database '{}'", - field_name, db_name - ); - Err(msg) - } - } - } - - fn optional(opt: &Option) -> T - where - T: Default, - { - opt.clone().unwrap_or_default() - } - let mut databases = Vec::with_capacity(input_config.databases.len()); - for db in input_config.databases { - if Uuid::parse_str(&db.generated_id).is_err() { - return Err(format!("Invalid UUID for database '{}'", db.name)); - } - - let username = match db.db_type { - DbType::Postgresql - | DbType::PostgresqlCluster - | DbType::Mysql - | DbType::Mariadb - | DbType::Mssql => required(&db.username, &db.name, "username")?, - _ => optional(&db.username), - }; - - let password = match db.db_type { - DbType::Postgresql - | DbType::PostgresqlCluster - | DbType::Mysql - | DbType::Mariadb - | DbType::Mssql => required(&db.password, &db.name, "password")?, - _ => optional(&db.password), - }; - - let host = match db.db_type { - DbType::Postgresql - | DbType::PostgresqlCluster - | DbType::Mysql - | DbType::Mariadb - | DbType::MongoDB - | DbType::Redis - | DbType::Firebird - | DbType::Valkey - | DbType::Mssql => required(&db.host, &db.name, "host")?, - DbType::Sqlite | DbType::DockerVolume => optional(&db.host), - }; - - let port = match db.db_type { - DbType::Postgresql - | DbType::PostgresqlCluster - | DbType::Mysql - | DbType::Mariadb - | DbType::MongoDB - | DbType::Redis - | DbType::Firebird - | DbType::Valkey - | DbType::Mssql => required(&db.port, &db.name, "port")?, - DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0), - }; - - let database_name = match db.db_type { - DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => { - optional(&db.database) - } - DbType::PostgresqlCluster => db - .database - .clone() - .unwrap_or_else(|| "postgres".to_string()), - _ => required(&db.database, &db.name, "database")?, - }; - - let path_val = match db.db_type { - DbType::Sqlite => required(&db.path, &db.name, "path")?, - _ => optional(&db.path), - }; - - let max_packet_size = match db.db_type { - DbType::Mysql | DbType::Mariadb => { - db.max_packet_size.unwrap_or_else(|| "512M".to_string()) - } - _ => String::new(), - }; - - let volume_name = match db.db_type { - DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?, - _ => optional(&db.volume_name), - }; - let container_name = db.container_name.clone(); - - databases.push(DatabaseConfig { - name: db.name, - database: database_name, - db_type: db.db_type, - username, - password, - host, - port, - generated_id: db.generated_id, - path: path_val, - max_packet_size, - volume_name, - container_name, - options: db.options.unwrap_or_default(), - }); + databases.push(build_config(db)?); } info!("Databases: {} instances loaded", databases.len()); Ok(DatabasesConfig { databases }) } + + /// Like `load`, but a missing or unreadable local file is not fatal: it logs a + /// warning and returns an empty set so the agent can still operate on + /// dashboard-defined databases. + pub fn load_optional(&self, file_path: Option<&str>) -> DatabasesConfig { + match self.load(file_path) { + Ok(cfg) => cfg, + Err(e) => { + tracing::warn!( + "Local databases config unavailable ({e}); continuing with dashboard-defined databases only" + ); + DatabasesConfig { databases: Vec::new() } + } + } + } } diff --git a/src/tests/services/config_tests.rs b/src/tests/services/config_tests.rs index 0936617..62b17e9 100644 --- a/src/tests/services/config_tests.rs +++ b/src/tests/services/config_tests.rs @@ -1,6 +1,7 @@ use crate::core::context::Context; use crate::services::api::ApiClient; use crate::services::config::ConfigService; +use crate::services::config::{build_config, DatabasesConfig, InputDatabaseConfig}; use crate::utils::edge_key::EdgeKey; use std::io::Write; use std::sync::Arc; @@ -264,3 +265,73 @@ fn docker_volume_requires_volume_name() { let err = service.load(Some(file.path().to_str().unwrap())).unwrap_err(); assert!(err.contains("volume_name"), "error was: {err}"); } + +#[test] +fn build_config_applies_type_defaults() { + let input: InputDatabaseConfig = serde_json::from_str( + r#"{ + "name": "cluster1", + "type": "postgresql-cluster", + "username": "postgres", + "password": "p", + "port": 5432, + "host": "localhost", + "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" + }"#, + ) + .unwrap(); + + let cfg = build_config(input).unwrap(); + assert_eq!(cfg.db_type.as_str(), "postgresql-cluster"); + assert_eq!(cfg.database, "postgres"); // cluster default +} + +#[test] +fn build_config_rejects_missing_required_field() { + let input: InputDatabaseConfig = serde_json::from_str( + r#"{ + "name": "pg", + "type": "postgresql", + "username": "postgres", + "port": 5432, + "host": "localhost", + "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" + }"#, + ) + .unwrap(); + + let err = build_config(input).unwrap_err(); + assert!(err.contains("password"), "unexpected error: {err}"); +} + +#[test] +fn load_optional_returns_empty_when_file_missing() { + let service = ConfigService::new(test_context()); + let cfg = service.load_optional(Some("/nonexistent/path/does-not-exist.json")); + assert!(cfg.databases.is_empty()); +} + +#[test] +fn databases_config_roundtrips_through_serde() { + let input: InputDatabaseConfig = serde_json::from_str( + r#"{ + "name": "pg", + "type": "postgresql", + "database": "app", + "username": "postgres", + "password": "secret", + "port": 5432, + "host": "localhost", + "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" + }"#, + ) + .unwrap(); + let cfg = build_config(input).unwrap(); + let wrapped = DatabasesConfig { databases: vec![cfg] }; + + let json = serde_json::to_string(&wrapped).unwrap(); + let back: DatabasesConfig = serde_json::from_str(&json).unwrap(); + assert_eq!(back.databases[0].name, "pg"); + assert_eq!(back.databases[0].db_type.as_str(), "postgresql"); + assert_eq!(back.databases[0].password, "secret"); +} From 4e2f29f4cab51d84b8b1f2ae3c274ab0aca2785c Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Thu, 13 Aug 2026 18:38:15 +0200 Subject: [PATCH 02/10] feat(status): decrypt dashboard config_ciphertext into resolved_config --- src/services/api/models/agent/status.rs | 8 ++ src/services/status.rs | 30 +++++++- src/tests/services/api_models_tests.rs | 97 +++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 1 deletion(-) diff --git a/src/services/api/models/agent/status.rs b/src/services/api/models/agent/status.rs index d8c4ea3..1a5aeab 100644 --- a/src/services/api/models/agent/status.rs +++ b/src/services/api/models/agent/status.rs @@ -1,5 +1,6 @@ #![allow(dead_code)] +use crate::services::config::DatabaseConfig; use crate::utils::deserializer::{deserialize_snake_case, string_or_number_to_string}; use serde::{Deserialize, Serialize}; use toml::Value; @@ -39,6 +40,13 @@ pub struct DatabaseStatus { pub storages_encrypted: Option, #[serde(default)] pub storages_ciphertext: Option, + #[serde(default)] + pub config_encrypted: Option, + #[serde(default)] + pub config_ciphertext: Option, + /// Filled in memory after decrypting `config_ciphertext`; never on the wire. + #[serde(skip)] + pub resolved_config: Option, pub encrypt: bool, pub data: DatabaseData, } diff --git a/src/services/status.rs b/src/services/status.rs index 1445588..df37f84 100644 --- a/src/services/status.rs +++ b/src/services/status.rs @@ -3,9 +3,10 @@ use crate::core::context::Context; use crate::domain::factory::DatabaseFactory; use crate::services::api::endpoints::status::DatabasePayload; +use crate::services::api::models::agent::status::DatabaseStatus; use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::api::models::agent::status::PingResult; -use crate::services::config::DatabaseConfig; +use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig}; use crate::settings::CONFIG; use crate::utils::file::decrypt_json_gcm; use futures_util::future::try_join_all; @@ -14,6 +15,29 @@ use std::error::Error; use std::sync::Arc; use tracing::info; +/// Decrypt a database's dashboard `config_ciphertext` (if present) into +/// `resolved_config`. AES-256-GCM envelope, same as `storages`. Errors are +/// returned so the caller can log-and-skip a single database. +pub fn resolve_dashboard_config( + status: &mut DatabaseStatus, + master_key_b64: &str, +) -> Result<(), String> { + if status.config_encrypted != Some(true) { + return Ok(()); + } + let ciphertext = status + .config_ciphertext + .as_deref() + .ok_or("config_encrypted set but config_ciphertext missing")?; + + let plaintext = decrypt_json_gcm(ciphertext, master_key_b64) + .map_err(|e| format!("Failed to decrypt config: {e}"))?; + let input: InputDatabaseConfig = serde_json::from_slice(&plaintext) + .map_err(|e| format!("Failed to parse decrypted config: {e}"))?; + status.resolved_config = Some(build_config(input)?); + Ok(()) +} + pub struct StatusService { ctx: Arc, client: Client, @@ -67,6 +91,10 @@ impl StatusService { db.storages = serde_json::from_slice::>(&plaintext) .map_err(|e| format!("Failed to parse decrypted storages: {e}"))?; } + + if let Err(e) = resolve_dashboard_config(db, &edge_key.master_key_b64) { + tracing::warn!("Skipping dashboard config for {}: {e}", db.generated_id); + } } Ok(result) } diff --git a/src/tests/services/api_models_tests.rs b/src/tests/services/api_models_tests.rs index 960334a..ea21856 100644 --- a/src/tests/services/api_models_tests.rs +++ b/src/tests/services/api_models_tests.rs @@ -148,3 +148,100 @@ fn database_status_encrypted_envelope() { assert_eq!(status.storages_encrypted, Some(true)); assert_eq!(status.storages_ciphertext.as_deref(), Some("AQIDBA==")); } + +#[test] +fn database_status_defaults_config_fields_absent() { + let json = r#"{ + "dbms": "postgresql", + "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681", + "encrypt": false, + "data": { "backup": { "action": false, "cron": null }, + "restore": { "action": false, "file": null, "metaFile": null, "size": null } } + }"#; + let status: crate::services::api::models::agent::status::DatabaseStatus = + serde_json::from_str(json).unwrap(); + assert_eq!(status.config_encrypted, None); + assert!(status.config_ciphertext.is_none()); + assert!(status.resolved_config.is_none()); +} + +#[test] +fn resolve_dashboard_config_decrypts_full_entry() { + use crate::services::status::resolve_dashboard_config; + use base64::{engine::general_purpose, Engine}; + + // 32-byte master key, base64 STANDARD (matches decrypt_json_gcm). + let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]); + + // Full agent-entry shape the dashboard encrypts. + let entry = r#"{ + "name": "Dashboard PG", + "type": "postgresql", + "database": "app", + "username": "postgres", + "password": "s3cret", + "port": 5432, + "host": "10.0.0.10", + "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" + }"#; + let ciphertext = encrypt_json_gcm(entry.as_bytes(), &master_key_b64); + + let mut status: crate::services::api::models::agent::status::DatabaseStatus = + serde_json::from_str( + r#"{ + "dbms": "postgresql", + "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681", + "encrypt": false, + "config_encrypted": true, + "config_ciphertext": "PLACEHOLDER", + "data": { "backup": { "action": false, "cron": null }, + "restore": { "action": false, "file": null, "metaFile": null, "size": null } } + }"#, + ) + .unwrap(); + status.config_ciphertext = Some(ciphertext); + + resolve_dashboard_config(&mut status, &master_key_b64).unwrap(); + + let cfg = status.resolved_config.expect("resolved"); + assert_eq!(cfg.name, "Dashboard PG"); + assert_eq!(cfg.password, "s3cret"); + assert_eq!(cfg.host, "10.0.0.10"); + assert_eq!(cfg.db_type.as_str(), "postgresql"); +} + +#[test] +fn resolve_dashboard_config_noop_when_not_encrypted() { + use crate::services::status::resolve_dashboard_config; + let mut status: crate::services::api::models::agent::status::DatabaseStatus = + serde_json::from_str( + r#"{ + "dbms": "postgresql", + "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681", + "encrypt": false, + "data": { "backup": { "action": false, "cron": null }, + "restore": { "action": false, "file": null, "metaFile": null, "size": null } } + }"#, + ) + .unwrap(); + resolve_dashboard_config(&mut status, "unused").unwrap(); + assert!(status.resolved_config.is_none()); +} + +// Test helper: mirrors the dashboard's encryptJsonGcm envelope +// base64(nonce(12) ‖ ciphertext ‖ tag(16)). +fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String { + use aes_gcm::aead::{Aead, KeyInit}; + use aes_gcm::{Aes256Gcm, Key, Nonce}; + use base64::{engine::general_purpose, Engine}; + + let key_bytes = general_purpose::STANDARD.decode(master_key_b64).unwrap(); + let key = Key::::try_from(key_bytes.as_slice()).unwrap(); + let cipher = Aes256Gcm::new(&key); + let nonce_bytes = [0u8; 12]; + let nonce = Nonce::try_from(&nonce_bytes[..]).unwrap(); + let ct = cipher.encrypt(&nonce, plaintext).unwrap(); + let mut data = nonce_bytes.to_vec(); + data.extend_from_slice(&ct); + general_purpose::STANDARD.encode(data) +} From b26ff818895ae2b2d5934115f776147e9a072426 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Thu, 13 Aug 2026 18:39:54 +0200 Subject: [PATCH 03/10] feat(dashboard_config): merge (dashboard-wins) + collect_configs --- src/services/dashboard_config.rs | 31 +++++++++++++++ src/services/mod.rs | 1 + src/tests/services/dashboard_config_tests.rs | 42 ++++++++++++++++++++ src/tests/services/mod.rs | 1 + 4 files changed, 75 insertions(+) create mode 100644 src/services/dashboard_config.rs create mode 100644 src/tests/services/dashboard_config_tests.rs diff --git a/src/services/dashboard_config.rs b/src/services/dashboard_config.rs new file mode 100644 index 0000000..a9342ad --- /dev/null +++ b/src/services/dashboard_config.rs @@ -0,0 +1,31 @@ +#![allow(dead_code)] + +use crate::services::api::models::agent::status::PingResult; +use crate::services::config::{DatabaseConfig, DatabasesConfig}; + +/// Merge local (file) databases with dashboard-provided ones. Dashboard wins on +/// a matching `generated_id`; otherwise dashboard entries are appended. Local +/// order is preserved. +pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> DatabasesConfig { + let mut databases: Vec = local.to_vec(); + for d in dashboard { + if let Some(slot) = databases + .iter_mut() + .find(|c| c.generated_id == d.generated_id) + { + *slot = d.clone(); + } else { + databases.push(d.clone()); + } + } + DatabasesConfig { databases } +} + +/// The authoritative dashboard set for this cycle: every returned database whose +/// `config_ciphertext` decrypted successfully into a `resolved_config`. +pub fn collect_configs(ping: &PingResult) -> Vec { + ping.databases + .iter() + .filter_map(|db| db.resolved_config.clone()) + .collect() +} diff --git a/src/services/mod.rs b/src/services/mod.rs index 260c3ab..e6c391a 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -2,6 +2,7 @@ pub mod api; pub mod backup; pub mod config; pub mod cron; +pub mod dashboard_config; pub mod restore; pub mod status; pub mod storage; diff --git a/src/tests/services/dashboard_config_tests.rs b/src/tests/services/dashboard_config_tests.rs new file mode 100644 index 0000000..b6eedf2 --- /dev/null +++ b/src/tests/services/dashboard_config_tests.rs @@ -0,0 +1,42 @@ +use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig}; +use crate::services::dashboard_config::merge; + +fn cfg(name: &str, gen_id: &str, host: &str) -> DatabaseConfig { + let json = format!( + r#"{{ "name": "{name}", "type": "postgresql", "database": "app", + "username": "u", "password": "p", "port": 5432, + "host": "{host}", "generated_id": "{gen_id}" }}"# + ); + let input: InputDatabaseConfig = serde_json::from_str(&json).unwrap(); + build_config(input).unwrap() +} + +const ID_A: &str = "16678159-ff7e-4c97-8c83-0adeff214681"; +const ID_B: &str = "16678124-ff7e-4c97-8c83-0adeff214681"; + +#[test] +fn merge_keeps_local_only_databases() { + let local = vec![cfg("local-a", ID_A, "local-host")]; + let merged = merge(&local, &[]); + assert_eq!(merged.databases.len(), 1); + assert_eq!(merged.databases[0].host, "local-host"); +} + +#[test] +fn merge_appends_dashboard_only_databases() { + let local = vec![cfg("local-a", ID_A, "local-host")]; + let dashboard = vec![cfg("dash-b", ID_B, "dash-host")]; + let merged = merge(&local, &dashboard); + assert_eq!(merged.databases.len(), 2); + assert!(merged.databases.iter().any(|d| d.generated_id == ID_B)); +} + +#[test] +fn merge_dashboard_wins_on_id_collision() { + let local = vec![cfg("local-a", ID_A, "local-host")]; + let dashboard = vec![cfg("dash-a", ID_A, "dash-host")]; + let merged = merge(&local, &dashboard); + assert_eq!(merged.databases.len(), 1); + assert_eq!(merged.databases[0].host, "dash-host"); // dashboard wins + assert_eq!(merged.databases[0].name, "dash-a"); +} diff --git a/src/tests/services/mod.rs b/src/tests/services/mod.rs index af4f3eb..b3a352c 100644 --- a/src/tests/services/mod.rs +++ b/src/tests/services/mod.rs @@ -1,3 +1,4 @@ mod api_models_tests; mod backup_uploader_tests; mod config_tests; +mod dashboard_config_tests; From 0a6eb6db22a52f09b7ce9096f4cb2bec45904095 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Thu, 13 Aug 2026 18:40:56 +0200 Subject: [PATCH 04/10] feat(dashboard_config): atomic cache load/persist --- src/services/dashboard_config.rs | 32 +++++++++++++++ src/tests/services/dashboard_config_tests.rs | 42 ++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/services/dashboard_config.rs b/src/services/dashboard_config.rs index a9342ad..8505861 100644 --- a/src/services/dashboard_config.rs +++ b/src/services/dashboard_config.rs @@ -2,6 +2,7 @@ use crate::services::api::models::agent::status::PingResult; use crate::services::config::{DatabaseConfig, DatabasesConfig}; +use std::path::Path; /// Merge local (file) databases with dashboard-provided ones. Dashboard wins on /// a matching `generated_id`; otherwise dashboard entries are appended. Local @@ -29,3 +30,34 @@ pub fn collect_configs(ping: &PingResult) -> Vec { .filter_map(|db| db.resolved_config.clone()) .collect() } + +/// Load the persisted dashboard set. A missing or corrupt file is treated as an +/// empty set (logged), so a bad cache can never crash startup — it is rebuilt +/// from the next `/status` response. +pub fn load_cache(path: &Path) -> Vec { + let contents = match std::fs::read_to_string(path) { + Ok(c) => c, + Err(_) => return Vec::new(), + }; + match serde_json::from_str::(&contents) { + Ok(cfg) => cfg.databases, + Err(e) => { + tracing::warn!("Dashboard cache at {:?} is corrupt ({e}); ignoring", path); + Vec::new() + } + } +} + +/// Persist the dashboard set atomically: write a sibling `*.tmp` then rename over +/// the target, so a crash mid-write can never leave a half-written cache. +pub fn persist_cache(path: &Path, databases: &[DatabaseConfig]) -> std::io::Result<()> { + let wrapper = DatabasesConfig { + databases: databases.to_vec(), + }; + let json = serde_json::to_string_pretty(&wrapper) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, json)?; + std::fs::rename(&tmp, path)?; + Ok(()) +} diff --git a/src/tests/services/dashboard_config_tests.rs b/src/tests/services/dashboard_config_tests.rs index b6eedf2..68027ad 100644 --- a/src/tests/services/dashboard_config_tests.rs +++ b/src/tests/services/dashboard_config_tests.rs @@ -1,5 +1,6 @@ use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig}; use crate::services::dashboard_config::merge; +use crate::services::dashboard_config::{load_cache, persist_cache}; fn cfg(name: &str, gen_id: &str, host: &str) -> DatabaseConfig { let json = format!( @@ -40,3 +41,44 @@ fn merge_dashboard_wins_on_id_collision() { assert_eq!(merged.databases[0].host, "dash-host"); // dashboard wins assert_eq!(merged.databases[0].name, "dash-a"); } + +#[test] +fn cache_roundtrips() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("dashboard_databases.json"); + + let dbs = vec![cfg("dash-a", ID_A, "dash-host")]; + persist_cache(&path, &dbs).unwrap(); + + let loaded = load_cache(&path); + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].generated_id, ID_A); + assert_eq!(loaded[0].host, "dash-host"); +} + +#[test] +fn load_cache_missing_file_is_empty() { + let loaded = load_cache(std::path::Path::new("/nonexistent/dashboard_databases.json")); + assert!(loaded.is_empty()); +} + +#[test] +fn load_cache_corrupt_file_is_empty() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("dashboard_databases.json"); + std::fs::write(&path, b"{ this is not valid json").unwrap(); + + let loaded = load_cache(&path); + assert!(loaded.is_empty()); +} + +#[test] +fn persist_cache_leaves_no_tmp_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("dashboard_databases.json"); + persist_cache(&path, &[cfg("dash-a", ID_A, "h")]).unwrap(); + + let tmp = path.with_extension("json.tmp"); + assert!(!tmp.exists(), "temp file should have been renamed away"); + assert!(path.exists()); +} From 044bf806332da754b291637f75b02fc0a03bb46d Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Thu, 13 Aug 2026 18:47:18 +0200 Subject: [PATCH 05/10] feat(agent): ingest dashboard databases via merged cycle + cache --- src/core/agent.rs | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/core/agent.rs b/src/core/agent.rs index bd20108..d296d48 100644 --- a/src/core/agent.rs +++ b/src/core/agent.rs @@ -2,13 +2,16 @@ use crate::core::context::Context; use crate::services::backup::BackupService; -use crate::services::config::ConfigService; +use crate::services::config::{ConfigService, DatabaseConfig}; use crate::services::cron::CronService; +use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache}; use crate::services::restore::RestoreService; use crate::services::status::StatusService; +use crate::settings::CONFIG; use crate::utils::common::BackupMethod; +use std::path::PathBuf; use std::sync::Arc; -use tracing::info; +use tracing::{error, info, warn}; pub struct Agent { ctx: Arc, @@ -17,6 +20,8 @@ pub struct Agent { cron_service: CronService, backup_service: BackupService, restore_service: RestoreService, + dashboard_cache: Vec, + cache_path: PathBuf, } impl Agent { @@ -28,6 +33,9 @@ impl Agent { let backup_service = BackupService::new(ctx.clone()); let restore_service = RestoreService::new(ctx.clone()); + let cache_path = PathBuf::from(&CONFIG.data_path).join("dashboard_databases.json"); + let dashboard_cache = load_cache(&cache_path); + Agent { ctx, config_service, @@ -35,19 +43,39 @@ impl Agent { cron_service, backup_service, restore_service, + dashboard_cache, + cache_path, } } pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box> { - let config = self.config_service.load(None)?; - let ping_result = self.status_service.ping(&config.databases).await?; + let local = self.config_service.load_optional(None); + + // Ping the merged set (local ∪ dashboard cache from the previous cycle) + // so the payload — and thus Ping Status — covers dashboard databases too. + let merged_in = merge(&local.databases, &self.dashboard_cache); + let ping_result = self.status_service.ping(&merged_in.databases).await?; + + // Reconcile the authoritative dashboard set from this response + // (full-state: additions, edits and deletions all self-apply) and persist it. + self.dashboard_cache = collect_configs(&ping_result); + if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) { + error!("Failed to persist dashboard cache: {e}"); + } + + // Rebuild the merged set including databases new in THIS response, so their + // backup/restore actions can run in the same cycle. + let merged = merge(&local.databases, &self.dashboard_cache); for db in ping_result.databases.iter() { - let database = config + let Some(database) = merged .databases .iter() .find(|cfg_db| cfg_db.generated_id == db.generated_id) - .unwrap(); + else { + warn!("No config for returned database {}; skipping", db.generated_id); + continue; + }; info!( "Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}", db.generated_id, db.data.backup.action, db.data.restore.action, database.name, @@ -59,14 +87,14 @@ impl Agent { .backup_service .dispatch( &db.generated_id, - &config, + &merged, method.clone(), &db.storages, db.encrypt, ) .await; } else if db.data.restore.action { - let _ = self.restore_service.dispatch(db, &config).await; + let _ = self.restore_service.dispatch(db, &merged).await; } } From 9d393a96a47e37e8e99b78e06b2e668426a4d1fb Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Thu, 13 Aug 2026 21:55:59 +0200 Subject: [PATCH 06/10] fix --- docker-compose.yml | 2 +- src/main.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bac575e..eab0968 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNmM4NWE3ODQtODRkMi00YzUyLTgzYmUtZTc2MDZkZjg2YjM5IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMGNjODEyYzktM2M4Ny00NmM3LWJlNjctOTI4MDljMmQ3N2NlIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 #DATABASES_CONFIG_FILE: "config.toml" diff --git a/src/main.rs b/src/main.rs index f92da4a..0414a61 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,7 +22,6 @@ async fn main() { eprintln!("Failed to clean locks on startup: {:?}", e); } - // Best-effort cleanup of ephemeral helper containers orphaned by a crash. match crate::domain::docker_volume::docker::client() { Ok(docker) => match crate::domain::docker_volume::docker::sweep_ephemeral(&docker).await { Ok(n) if n > 0 => tracing::info!("Removed {n} orphaned ephemeral helper container(s)"), From ca294e968c261d2b2c491372c71f872ae180cee6 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Thu, 13 Aug 2026 22:02:17 +0200 Subject: [PATCH 07/10] fix --- src/core/agent.rs | 6 ------ src/services/config.rs | 16 +++++----------- src/services/dashboard_config.rs | 10 ---------- src/services/status.rs | 3 --- src/tests/services/api_models_tests.rs | 2 -- src/tests/services/config_tests.rs | 4 ---- 6 files changed, 5 insertions(+), 36 deletions(-) diff --git a/src/core/agent.rs b/src/core/agent.rs index d296d48..9c78d4e 100644 --- a/src/core/agent.rs +++ b/src/core/agent.rs @@ -51,20 +51,14 @@ impl Agent { pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box> { let local = self.config_service.load_optional(None); - // Ping the merged set (local ∪ dashboard cache from the previous cycle) - // so the payload — and thus Ping Status — covers dashboard databases too. let merged_in = merge(&local.databases, &self.dashboard_cache); let ping_result = self.status_service.ping(&merged_in.databases).await?; - // Reconcile the authoritative dashboard set from this response - // (full-state: additions, edits and deletions all self-apply) and persist it. self.dashboard_cache = collect_configs(&ping_result); if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) { error!("Failed to persist dashboard cache: {e}"); } - // Rebuild the merged set including databases new in THIS response, so their - // backup/restore actions can run in the same cycle. let merged = merge(&local.databases, &self.dashboard_cache); for db in ping_result.databases.iter() { diff --git a/src/services/config.rs b/src/services/config.rs index 9142c78..b1a25cd 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -238,19 +238,13 @@ impl ConfigService { info!("Databases: {} instances loaded", databases.len()); Ok(DatabasesConfig { databases }) } - - /// Like `load`, but a missing or unreadable local file is not fatal: it logs a - /// warning and returns an empty set so the agent can still operate on - /// dashboard-defined databases. + pub fn load_optional(&self, file_path: Option<&str>) -> DatabasesConfig { - match self.load(file_path) { - Ok(cfg) => cfg, - Err(e) => { - tracing::warn!( + self.load(file_path).unwrap_or_else(|e| { + tracing::warn!( "Local databases config unavailable ({e}); continuing with dashboard-defined databases only" ); - DatabasesConfig { databases: Vec::new() } - } - } + DatabasesConfig { databases: Vec::new() } + }) } } diff --git a/src/services/dashboard_config.rs b/src/services/dashboard_config.rs index 8505861..c6ef9b1 100644 --- a/src/services/dashboard_config.rs +++ b/src/services/dashboard_config.rs @@ -4,9 +4,6 @@ use crate::services::api::models::agent::status::PingResult; use crate::services::config::{DatabaseConfig, DatabasesConfig}; use std::path::Path; -/// Merge local (file) databases with dashboard-provided ones. Dashboard wins on -/// a matching `generated_id`; otherwise dashboard entries are appended. Local -/// order is preserved. pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> DatabasesConfig { let mut databases: Vec = local.to_vec(); for d in dashboard { @@ -22,8 +19,6 @@ pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> Database DatabasesConfig { databases } } -/// The authoritative dashboard set for this cycle: every returned database whose -/// `config_ciphertext` decrypted successfully into a `resolved_config`. pub fn collect_configs(ping: &PingResult) -> Vec { ping.databases .iter() @@ -31,9 +26,6 @@ pub fn collect_configs(ping: &PingResult) -> Vec { .collect() } -/// Load the persisted dashboard set. A missing or corrupt file is treated as an -/// empty set (logged), so a bad cache can never crash startup — it is rebuilt -/// from the next `/status` response. pub fn load_cache(path: &Path) -> Vec { let contents = match std::fs::read_to_string(path) { Ok(c) => c, @@ -48,8 +40,6 @@ pub fn load_cache(path: &Path) -> Vec { } } -/// Persist the dashboard set atomically: write a sibling `*.tmp` then rename over -/// the target, so a crash mid-write can never leave a half-written cache. pub fn persist_cache(path: &Path, databases: &[DatabaseConfig]) -> std::io::Result<()> { let wrapper = DatabasesConfig { databases: databases.to_vec(), diff --git a/src/services/status.rs b/src/services/status.rs index df37f84..8779adb 100644 --- a/src/services/status.rs +++ b/src/services/status.rs @@ -15,9 +15,6 @@ use std::error::Error; use std::sync::Arc; use tracing::info; -/// Decrypt a database's dashboard `config_ciphertext` (if present) into -/// `resolved_config`. AES-256-GCM envelope, same as `storages`. Errors are -/// returned so the caller can log-and-skip a single database. pub fn resolve_dashboard_config( status: &mut DatabaseStatus, master_key_b64: &str, diff --git a/src/tests/services/api_models_tests.rs b/src/tests/services/api_models_tests.rs index ea21856..a4f8e87 100644 --- a/src/tests/services/api_models_tests.rs +++ b/src/tests/services/api_models_tests.rs @@ -228,8 +228,6 @@ fn resolve_dashboard_config_noop_when_not_encrypted() { assert!(status.resolved_config.is_none()); } -// Test helper: mirrors the dashboard's encryptJsonGcm envelope -// base64(nonce(12) ‖ ciphertext ‖ tag(16)). fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String { use aes_gcm::aead::{Aead, KeyInit}; use aes_gcm::{Aes256Gcm, Key, Nonce}; diff --git a/src/tests/services/config_tests.rs b/src/tests/services/config_tests.rs index 62b17e9..6ceb172 100644 --- a/src/tests/services/config_tests.rs +++ b/src/tests/services/config_tests.rs @@ -7,10 +7,6 @@ use std::io::Write; use std::sync::Arc; use tempfile::NamedTempFile; -// `ConfigService::load` never touches `self.ctx` on the `Some(file_path)` path, -// so the values here don't matter — but `Context::new()` panics without an -// `EDGE_KEY` env var, so build the struct directly (mirrors -// backup_uploader_tests.rs's `ctx_pointing_at`). fn test_context() -> Arc { Arc::new(Context { edge_key: EdgeKey { From cf9a59a13807eed26058ebdcd089b1d4ac35604d Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Tue, 18 Aug 2026 13:07:22 +0200 Subject: [PATCH 08/10] fix: config.rs --- src/services/config.rs | 46 ++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/services/config.rs b/src/services/config.rs index b1a25cd..b58ae65 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -118,34 +118,54 @@ pub fn build_config(db: InputDatabaseConfig) -> Result { } let username = match db.db_type { - DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb + DbType::Postgresql + | DbType::PostgresqlCluster + | DbType::Mysql + | DbType::Mariadb | DbType::Mssql => required(&db.username, &db.name, "username")?, _ => optional(&db.username), }; let password = match db.db_type { - DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb + DbType::Postgresql + | DbType::PostgresqlCluster + | DbType::Mysql + | DbType::Mariadb | DbType::Mssql => required(&db.password, &db.name, "password")?, _ => optional(&db.password), }; let host = match db.db_type { - DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb - | DbType::MongoDB | DbType::Redis | DbType::Firebird | DbType::Valkey | DbType::Mssql => { - required(&db.host, &db.name, "host")? - } + DbType::Postgresql + | DbType::PostgresqlCluster + | DbType::Mysql + | DbType::Mariadb + | DbType::MongoDB + | DbType::Redis + | DbType::Firebird + | DbType::Valkey + | DbType::Mssql => required(&db.host, &db.name, "host")?, DbType::Sqlite | DbType::DockerVolume => optional(&db.host), }; + let port = match db.db_type { - DbType::Postgresql | DbType::PostgresqlCluster | DbType::Mysql | DbType::Mariadb - | DbType::MongoDB | DbType::Redis | DbType::Firebird | DbType::Valkey | DbType::Mssql => { - required(&db.port, &db.name, "port")? - } - DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0), + DbType::Postgresql + | DbType::PostgresqlCluster + | DbType::Mysql + | DbType::Mariadb + | DbType::Redis + | DbType::Firebird + | DbType::Valkey + | DbType::Mssql => required(&db.port, &db.name, "port")?, + DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0), }; + let database_name = match db.db_type { DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => { optional(&db.database) } - DbType::PostgresqlCluster => db.database.clone().unwrap_or_else(|| "postgres".to_string()), + DbType::PostgresqlCluster => db + .database + .clone() + .unwrap_or_else(|| "postgres".to_string()), _ => required(&db.database, &db.name, "database")?, }; let path_val = match db.db_type { @@ -238,7 +258,7 @@ impl ConfigService { info!("Databases: {} instances loaded", databases.len()); Ok(DatabasesConfig { databases }) } - + pub fn load_optional(&self, file_path: Option<&str>) -> DatabasesConfig { self.load(file_path).unwrap_or_else(|e| { tracing::warn!( From 046d593e2bf7b1c613ce2512b9dcf78b99ac57c1 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Tue, 18 Aug 2026 18:20:21 +0200 Subject: [PATCH 09/10] fix --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index eab0968..8764d2b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMGNjODEyYzktM2M4Ny00NmM3LWJlNjctOTI4MDljMmQ3N2NlIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMGNlNjVjMDQtMDJiOS00YjMzLTk5MzYtMzIzYTFhOGU4MTk3IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 #DATABASES_CONFIG_FILE: "config.toml" From 90941ea67d05f1846fe3a22e200258c6d8ae2a0e Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Thu, 20 Aug 2026 13:41:14 +0200 Subject: [PATCH 10/10] fix --- docker-compose.yml | 2 +- src/services/storage/providers/s3/mod.rs | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8764d2b..d254472 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: LOG: debug TZ: "Europe/Paris" # TMPDIR: /scratch - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMGNlNjVjMDQtMDJiOS00YjMzLTk5MzYtMzIzYTFhOGU4MTk3IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZDY4MzU2MTQtNzE2NC00OTQ4LWJlZjMtMTlkZDc5NGQzYmRhIiwibWFzdGVyS2V5QjY0IjoiV2NiM0pQQkVTaFBjRjg5UXZwRVJuamU4NGZmak1kNm4vS2dJOUpjMCtmVT0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 #DATABASES_CONFIG_FILE: "config.toml" diff --git a/src/services/storage/providers/s3/mod.rs b/src/services/storage/providers/s3/mod.rs index 22847ca..4d79f24 100644 --- a/src/services/storage/providers/s3/mod.rs +++ b/src/services/storage/providers/s3/mod.rs @@ -137,9 +137,6 @@ impl StorageProvider for S3Provider { .credentials_provider(credentials) .region(region) .force_path_style(true) - // S3-compatible endpoints (MinIO, Garage, RustFS, Synology, ...) reject the - // default CRC32 integrity checksums the SDK attaches to multipart uploads. - // Only send checksums when the operation actually requires them. .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) .endpoint_url(endpoint) .behavior_version(BehaviorVersion::latest())