feat: add corex-storage crate for unified storage abstraction

- Introduced corex-storage crate with support for local disk, S3-compatible, and Google Cloud Storage backends.
- Implemented StorageDriver trait for various storage backends.
- Added LocalFileStorage for local disk operations.
- Added S3Storage for S3-compatible object storage with multipart upload support.
- Added GcsStorage for Google Cloud Storage operations.
- Included error handling for storage operations.
- Added tests for each storage backend to ensure functionality.
- Created README.md for documentation and usage examples.
- Added Apache and MIT licenses for open-source compliance.
This commit is contained in:
asepharyana
2026-08-28 22:24:18 +07:00
parent 6a4b2b8f54
commit db4f277336
47 changed files with 0 additions and 0 deletions
+191
View File
@@ -0,0 +1,191 @@
//! Runtime configuration hot-reload (feature `hot-reload`).
//!
//! [`DynamicConfig`] holds a typed configuration value behind an `RwLock`,
//! optionally watching a set of source files with `notify` and re-running a
//! caller-supplied reload closure whenever they change. Subscribers can await
//! a [`tokio::sync::broadcast::Receiver`] to react to a successful reload.
use std::path::PathBuf;
use std::sync::{Arc, RwLock};
use std::time::{Duration, Instant};
use notify::{RecommendedWatcher, RecursiveMode, Watcher};
use tokio::sync::broadcast;
use crate::{Config, ConfigError};
/// A hot-reloadable, thread-safe configuration handle.
pub struct DynamicConfig<T> {
inner: Arc<RwLock<T>>,
tx: broadcast::Sender<()>,
}
impl<T: Config + Clone> Clone for DynamicConfig<T> {
fn clone(&self) -> Self {
Self {
inner: Arc::clone(&self.inner),
tx: self.tx.clone(),
}
}
}
impl<T: Config + Clone> DynamicConfig<T> {
/// Wraps an already-loaded value with no file watching.
pub fn new(initial: T) -> Self {
let (tx, _rx) = broadcast::channel(16);
Self {
inner: Arc::new(RwLock::new(initial)),
tx,
}
}
/// Returns a clone of the current configuration snapshot.
pub fn get(&self) -> T {
self.inner
.read()
.expect("corex-config: RwLock poisoned")
.clone()
}
/// Replaces the current value and notifies subscribers.
pub fn set(&self, new: T) {
*self.inner.write().expect("corex-config: RwLock poisoned") = new;
let _ = self.tx.send(());
}
/// Subscribes to change notifications (fired after every successful
/// [`DynamicConfig::set`] or file-triggered reload).
pub fn subscribe(&self) -> broadcast::Receiver<()> {
self.tx.subscribe()
}
/// Loads the initial value via `reload`, then watches `paths` for
/// modifications and re-runs `reload` on each change (debounced), atomically
/// swapping in the result on success. A failed reload is logged via
/// `tracing::error!` and the previous value is retained.
///
/// The underlying OS file watcher lives on a dedicated background thread
/// for the lifetime of the process; there is currently no explicit
/// "unwatch" — construct one `DynamicConfig` per watched file set.
pub fn watch_files<F>(paths: Vec<PathBuf>, reload: F) -> Result<Self, ConfigError>
where
F: Fn() -> Result<T, ConfigError> + Send + Sync + 'static,
{
Self::watch_files_debounced(paths, reload, Duration::from_millis(50))
}
/// Same as [`DynamicConfig::watch_files`] with an explicit debounce
/// window (the minimum time between two applied reloads).
pub fn watch_files_debounced<F>(
paths: Vec<PathBuf>,
reload: F,
debounce: Duration,
) -> Result<Self, ConfigError>
where
F: Fn() -> Result<T, ConfigError> + Send + Sync + 'static,
{
let initial = reload()?;
let config = Self::new(initial);
let inner = Arc::clone(&config.inner);
let tx = config.tx.clone();
let (raw_tx, raw_rx) = std::sync::mpsc::channel();
let mut watcher: RecommendedWatcher = notify::recommended_watcher(move |res| {
let _ = raw_tx.send(res);
})
.map_err(|e| ConfigError::Watch(e.to_string()))?;
for path in &paths {
watcher
.watch(path, RecursiveMode::NonRecursive)
.map_err(|e| ConfigError::Watch(e.to_string()))?;
}
std::thread::Builder::new()
.name("corex-config-watch".into())
.spawn(move || {
// Keep the watcher alive for the life of this thread.
let _watcher = watcher;
let mut last_applied = Instant::now() - debounce;
for event in raw_rx {
if event.is_err() {
continue;
}
if last_applied.elapsed() < debounce {
continue;
}
match reload() {
Ok(new) => {
*inner.write().expect("corex-config: RwLock poisoned") = new;
let _ = tx.send(());
last_applied = Instant::now();
}
Err(e) => {
tracing::error!("corex-config: hot-reload failed: {e}");
}
}
}
})
.map_err(|e| ConfigError::Watch(e.to_string()))?;
Ok(config)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ConfigLoader;
use serde::Deserialize;
use std::io::Write;
#[derive(Debug, Deserialize, Clone, PartialEq)]
struct Cfg {
value: u32,
}
#[test]
fn set_and_get_roundtrip() {
let cfg = DynamicConfig::new(Cfg { value: 1 });
assert_eq!(cfg.get(), Cfg { value: 1 });
cfg.set(Cfg { value: 2 });
assert_eq!(cfg.get(), Cfg { value: 2 });
}
#[tokio::test]
async fn subscribe_receives_change_notification() {
let cfg = DynamicConfig::new(Cfg { value: 1 });
let mut rx = cfg.subscribe();
cfg.set(Cfg { value: 9 });
rx.recv().await.expect("change notification");
assert_eq!(cfg.get().value, 9);
}
#[tokio::test]
async fn watch_files_reloads_on_change() {
let mut file = tempfile::Builder::new().suffix(".json").tempfile().unwrap();
write!(file, r#"{{"value": 1}}"#).unwrap();
let path = file.path().to_path_buf();
let reload_path = path.clone();
let cfg = DynamicConfig::<Cfg>::watch_files_debounced(
vec![path.clone()],
move || {
ConfigLoader::new()
.merge_file(&reload_path)
.and_then(|l| l.build())
},
Duration::from_millis(10),
)
.expect("watch setup");
assert_eq!(cfg.get().value, 1);
let mut rx = cfg.subscribe();
// Rewrite the file to trigger a reload.
std::fs::write(&path, r#"{"value": 42}"#).unwrap();
// Wait (with a generous timeout) for the watcher thread to notice.
let result = tokio::time::timeout(Duration::from_secs(5), rx.recv()).await;
assert!(result.is_ok(), "expected a reload notification within 5s");
assert_eq!(cfg.get().value, 42);
}
}
+32
View File
@@ -0,0 +1,32 @@
//! Shared error type for corex-config.
/// Errors surfaced while loading or reloading configuration.
#[non_exhaustive]
#[derive(Debug)]
pub enum ConfigError {
/// An I/O error reading a config source (file not found, permissions, ...).
Io(String),
/// A source could not be parsed (malformed YAML/JSON/TOML).
Parse(String),
/// The merged configuration could not be deserialized into the target type.
Deserialize(String),
/// The requested file extension has no registered loader (feature not
/// enabled, or unsupported format).
UnsupportedFormat(String),
/// Hot-reload setup failed (e.g. the file watcher could not be installed).
Watch(String),
}
impl std::fmt::Display for ConfigError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Io(s) => write!(f, "config io error: {s}"),
Self::Parse(s) => write!(f, "config parse error: {s}"),
Self::Deserialize(s) => write!(f, "config deserialize error: {s}"),
Self::UnsupportedFormat(s) => write!(f, "unsupported config format: {s}"),
Self::Watch(s) => write!(f, "config watch error: {s}"),
}
}
}
impl std::error::Error for ConfigError {}
+53
View File
@@ -0,0 +1,53 @@
//! # corex-config
//!
//! A type-safe, dynamic configuration engine: load environment variables,
//! `.env` files, YAML, JSON, or TOML directly into a typed Rust struct, with
//! optional runtime hot-reload.
//!
//! - [`ConfigLoader`] merges any number of file/env sources into a single
//! typed value (files first, environment variables override).
//! - [`DynamicConfig`] wraps a loaded value behind an `RwLock`, optionally
//! watching its source files and swapping in a freshly reloaded value when
//! they change, broadcasting the change to subscribers.
//!
//! ## Example
//!
//! ```no_run
//! use serde::Deserialize;
//! use corex_config::ConfigLoader;
//!
//! #[derive(Debug, Deserialize, Clone)]
//! struct AppConfig {
//! port: u16,
//! database: DatabaseConfig,
//! }
//! #[derive(Debug, Deserialize, Clone)]
//! struct DatabaseConfig {
//! url: String,
//! }
//!
//! let config: AppConfig = ConfigLoader::new()
//! .merge_file("config.yaml".as_ref())
//! .unwrap()
//! .merge_env("APP")
//! .build()
//! .unwrap();
//! ```
pub mod error;
pub mod loader;
#[cfg(feature = "hot-reload")]
pub mod dynamic;
pub use error::ConfigError;
pub use loader::ConfigLoader;
#[cfg(feature = "hot-reload")]
pub use dynamic::DynamicConfig;
/// Marker trait for types loadable by [`ConfigLoader`].
///
/// Blanket-implemented for any `Deserialize + Send + Sync + 'static`.
pub trait Config: for<'de> serde::Deserialize<'de> + Send + Sync + 'static {}
impl<T> Config for T where T: for<'de> serde::Deserialize<'de> + Send + Sync + 'static {}
+348
View File
@@ -0,0 +1,348 @@
//! Merges file and environment sources into a typed configuration value.
use std::marker::PhantomData;
use std::path::Path;
use serde_json::{Map, Value};
use crate::{Config, ConfigError};
/// Builds a typed configuration value from any number of sources.
///
/// Sources are merged in the order they're added; later sources override
/// earlier ones at the leaf level (objects are merged recursively, not
/// replaced wholesale). The conventional order is: defaults, then files
/// (base -> environment-specific), then environment variables (highest
/// priority, for secrets/overrides).
pub struct ConfigLoader<T> {
value: Value,
_marker: PhantomData<fn() -> T>,
}
impl<T> Default for ConfigLoader<T> {
fn default() -> Self {
Self {
value: Value::Object(Map::new()),
_marker: PhantomData,
}
}
}
impl<T: Config> ConfigLoader<T> {
/// Builds an empty loader.
pub fn new() -> Self {
Self::default()
}
/// Merges a JSON value directly (useful for defaults / tests).
pub fn merge_value(mut self, value: Value) -> Self {
deep_merge(&mut self.value, value);
self
}
/// Reads `path`, parses it by extension (`.yaml`/`.yml`, `.json`,
/// `.toml`), and merges the result.
pub fn merge_file(mut self, path: &Path) -> Result<Self, ConfigError> {
let contents = std::fs::read_to_string(path)
.map_err(|e| ConfigError::Io(format!("{}: {e}", path.display())))?;
let parsed = parse_by_extension(path, &contents)?;
deep_merge(&mut self.value, parsed);
Ok(self)
}
/// Loads a `.env`-style file into the process environment (does not merge
/// into the config value directly — call [`Self::merge_env`] afterward to
/// pick the variables up).
#[cfg(feature = "env")]
pub fn load_dotenv(self, path: &Path) -> Result<Self, ConfigError> {
dotenvy::from_path(path).map_err(|e| ConfigError::Io(e.to_string()))?;
Ok(self)
}
/// Merges environment variables whose name starts with `prefix` (an
/// underscore is inserted between the prefix and the field name if not
/// already present). Nested fields use `__` as a separator, e.g.
/// `APP_DATABASE__URL` maps to `{ "database": { "url": ... } }`.
///
/// Values are parsed as JSON scalars when possible (`true`/`false`,
/// integers, floats), otherwise kept as strings.
pub fn merge_env(mut self, prefix: &str) -> Self {
let collected = collect_env(prefix);
deep_merge(&mut self.value, Value::Object(collected));
self
}
/// Deserializes the merged value into `T`.
pub fn build(self) -> Result<T, ConfigError> {
serde_json::from_value(self.value).map_err(|e| ConfigError::Deserialize(e.to_string()))
}
/// Returns the current merged value without deserializing, for
/// inspection/debugging.
pub fn peek(&self) -> &Value {
&self.value
}
}
/// Parses `contents` according to `path`'s extension.
fn parse_by_extension(path: &Path, contents: &str) -> Result<Value, ConfigError> {
let ext = path
.extension()
.and_then(|e| e.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
match ext.as_str() {
#[cfg(feature = "yaml")]
"yaml" | "yml" => serde_yaml::from_str(contents)
.map_err(|e| ConfigError::Parse(e.to_string()))
.map(yaml_to_json),
"json" => serde_json::from_str(contents).map_err(|e| ConfigError::Parse(e.to_string())),
#[cfg(feature = "toml")]
"toml" => {
let v: toml::Value =
toml::from_str(contents).map_err(|e| ConfigError::Parse(e.to_string()))?;
Ok(toml_to_json(v))
}
other => Err(ConfigError::UnsupportedFormat(format!(
"no loader registered for `.{other}` (path: {})",
path.display()
))),
}
}
#[cfg(feature = "yaml")]
fn yaml_to_json(v: serde_yaml::Value) -> Value {
// Round-trip through serde_json for a uniform merge representation.
serde_json::to_value(v).unwrap_or(Value::Null)
}
#[cfg(feature = "toml")]
fn toml_to_json(v: toml::Value) -> Value {
serde_json::to_value(v).unwrap_or(Value::Null)
}
/// Deep-merges `overlay` into `base`; scalars and arrays in `overlay` replace
/// `base`, objects are merged key-by-key recursively.
fn deep_merge(base: &mut Value, overlay: Value) {
match (base, overlay) {
(Value::Object(base_map), Value::Object(overlay_map)) => {
for (k, v) in overlay_map {
match base_map.get_mut(&k) {
Some(existing) => deep_merge(existing, v),
None => {
base_map.insert(k, v);
}
}
}
}
(base_slot, overlay_value) => {
*base_slot = overlay_value;
}
}
}
/// Collects environment variables with `prefix` into a nested JSON object.
///
/// `PREFIX_FOO` -> `{"foo": ...}`; `PREFIX_FOO__BAR` -> `{"foo": {"bar": ...}}`.
fn collect_env(prefix: &str) -> Map<String, Value> {
let mut root = Map::new();
let full_prefix = if prefix.ends_with('_') {
prefix.to_string()
} else {
format!("{prefix}_")
};
for (key, raw_value) in std::env::vars() {
let Some(rest) = key.strip_prefix(&full_prefix) else {
continue;
};
if rest.is_empty() {
continue;
}
let path: Vec<String> = rest.split("__").map(|s| s.to_ascii_lowercase()).collect();
insert_nested(&mut root, &path, coerce_scalar(&raw_value));
}
root
}
fn insert_nested(root: &mut Map<String, Value>, path: &[String], value: Value) {
if path.len() == 1 {
root.insert(path[0].clone(), value);
return;
}
let entry = root
.entry(path[0].clone())
.or_insert_with(|| Value::Object(Map::new()));
if let Value::Object(nested) = entry {
insert_nested(nested, &path[1..], value);
}
}
/// Parses `s` as a JSON scalar (`bool`/number) if possible, else keeps it as
/// a string.
fn coerce_scalar(s: &str) -> Value {
if let Ok(b) = s.parse::<bool>() {
return Value::Bool(b);
}
if let Ok(i) = s.parse::<i64>() {
return Value::Number(i.into());
}
if let Ok(f) = s.parse::<f64>() {
if let Some(n) = serde_json::Number::from_f64(f) {
return Value::Number(n);
}
}
Value::String(s.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
use serde::Deserialize;
#[derive(Debug, Deserialize, PartialEq)]
struct Db {
url: String,
pool_size: u32,
}
#[derive(Debug, Deserialize, PartialEq)]
struct AppConfig {
port: u16,
debug: bool,
database: Db,
}
#[test]
fn merge_value_builds_typed_struct() {
let value = serde_json::json!({
"port": 8080,
"debug": true,
"database": { "url": "postgres://x", "pool_size": 10 }
});
let cfg: AppConfig = ConfigLoader::new().merge_value(value).build().unwrap();
assert_eq!(
cfg,
AppConfig {
port: 8080,
debug: true,
database: Db {
url: "postgres://x".into(),
pool_size: 10
}
}
);
}
#[test]
fn deep_merge_overrides_nested_leaf_only() {
let base = serde_json::json!({ "port": 8080, "debug": false, "database": { "url": "a", "pool_size": 5 } });
let overlay = serde_json::json!({ "database": { "pool_size": 20 } });
let cfg: AppConfig = ConfigLoader::new()
.merge_value(base)
.merge_value(overlay)
.build()
.unwrap();
assert_eq!(cfg.database.pool_size, 20);
assert_eq!(cfg.database.url, "a"); // untouched sibling field preserved
assert_eq!(cfg.port, 8080);
}
#[test]
fn env_merge_maps_nested_and_types() {
// Safe because this test owns unique env var names.
std::env::set_var("CFGTEST_PORT", "9090");
std::env::set_var("CFGTEST_DEBUG", "true");
std::env::set_var("CFGTEST_DATABASE__URL", "redis://y");
std::env::set_var("CFGTEST_DATABASE__POOL_SIZE", "42");
let cfg: AppConfig = ConfigLoader::new().merge_env("CFGTEST").build().unwrap();
assert_eq!(cfg.port, 9090);
assert!(cfg.debug);
assert_eq!(cfg.database.url, "redis://y");
assert_eq!(cfg.database.pool_size, 42);
std::env::remove_var("CFGTEST_PORT");
std::env::remove_var("CFGTEST_DEBUG");
std::env::remove_var("CFGTEST_DATABASE__URL");
std::env::remove_var("CFGTEST_DATABASE__POOL_SIZE");
}
#[test]
fn env_overrides_file_value() {
std::env::set_var("CFGTEST2_PORT", "7000");
let base = serde_json::json!({ "port": 1, "debug": false, "database": { "url": "a", "pool_size": 1 } });
let cfg: AppConfig = ConfigLoader::new()
.merge_value(base)
.merge_env("CFGTEST2")
.build()
.unwrap();
assert_eq!(cfg.port, 7000);
std::env::remove_var("CFGTEST2_PORT");
}
#[cfg(feature = "yaml")]
#[test]
fn merge_yaml_file() {
use std::io::Write;
let mut file = tempfile::Builder::new().suffix(".yaml").tempfile().unwrap();
writeln!(
file,
"port: 3000\ndebug: false\ndatabase:\n url: sqlite://mem\n pool_size: 3\n"
)
.unwrap();
let cfg: AppConfig = ConfigLoader::new()
.merge_file(file.path())
.unwrap()
.build()
.unwrap();
assert_eq!(cfg.port, 3000);
assert_eq!(cfg.database.url, "sqlite://mem");
}
#[test]
fn merge_json_file() {
use std::io::Write;
let mut file = tempfile::Builder::new().suffix(".json").tempfile().unwrap();
write!(
file,
r#"{{"port": 4000, "debug": true, "database": {{"url": "mysql://x", "pool_size": 7}}}}"#
)
.unwrap();
let cfg: AppConfig = ConfigLoader::new()
.merge_file(file.path())
.unwrap()
.build()
.unwrap();
assert_eq!(cfg.port, 4000);
assert_eq!(cfg.database.pool_size, 7);
}
#[cfg(feature = "toml")]
#[test]
fn merge_toml_file() {
use std::io::Write;
let mut file = tempfile::Builder::new().suffix(".toml").tempfile().unwrap();
writeln!(
file,
"port = 5000\ndebug = false\n\n[database]\nurl = \"file://local\"\npool_size = 2\n"
)
.unwrap();
let cfg: AppConfig = ConfigLoader::new()
.merge_file(file.path())
.unwrap()
.build()
.unwrap();
assert_eq!(cfg.port, 5000);
assert_eq!(cfg.database.pool_size, 2);
}
#[test]
fn unsupported_extension_errors() {
let file = tempfile::Builder::new().suffix(".ini").tempfile().unwrap();
std::fs::write(file.path(), "unused").unwrap();
let result: Result<AppConfig, _> = ConfigLoader::new()
.merge_file(file.path())
.and_then(|l| l.build());
assert!(matches!(result, Err(ConfigError::UnsupportedFormat(_))));
}
}