fix: clippy clean for round-2 (pipeline module export, lint cleanup)
This commit is contained in:
@@ -0,0 +1,24 @@
|
|||||||
|
# Implementation Spec: Round 2
|
||||||
|
|
||||||
|
## New Features
|
||||||
|
|
||||||
|
### 1. ServiceBuilder (mytheclipse-core)
|
||||||
|
File: `crates/mytheclipse/src/service_builder.rs`
|
||||||
|
- Builder that wraps async operations with retry + circuit breaker + timeout + rate limiter
|
||||||
|
- Fluent API: `.retry(config)`, `.circuit(config)`, `.timeout(dur)`, `.rate(rate, burst)`, `.concurrency(max)`, `.run(fut)`
|
||||||
|
- Feature gate: `resiliency` (uses existing retry/CircuitBreaker/timeout primitives)
|
||||||
|
- Integrates with metrics: records retries, circuit events, timeouts
|
||||||
|
|
||||||
|
### 2. DistributedLock (mytheclipse-core)
|
||||||
|
File: `crates/mytheclipse/src/dlock.rs`
|
||||||
|
- `DistributedLock` trait: `acquire(timeout)`, `release()`, `extend(lease_dur)`
|
||||||
|
- `InProcDistributedLock` impl using tokio Mutex + lease time tracking
|
||||||
|
- `RedisLock` impl (feature `redis`) — Redis SETNX with PX expiry
|
||||||
|
- Feature gate: `lifecycle` (uses existing leader election infra)
|
||||||
|
|
||||||
|
### 3. StreamingPipeline (mytheclipse-queue)
|
||||||
|
File: `crates/mytheclipse-queue/src/pipeline.rs`
|
||||||
|
- Pipe stages: `Stage<Input, Output>` trait with async `process(item) -> Output`
|
||||||
|
- Pipeline: `add_stage(impl Stage)`, `run(input_stream)`, `collect()`
|
||||||
|
- Backpressure: bounded channel between stages
|
||||||
|
- Feature gate: `in-memory` (uses tokio + std)
|
||||||
@@ -5,7 +5,6 @@ use axum::{
|
|||||||
Router,
|
Router,
|
||||||
};
|
};
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
/// A pre-configured HTTP server with health check and metrics endpoints.
|
/// A pre-configured HTTP server with health check and metrics endpoints.
|
||||||
pub struct HttpServer {
|
pub struct HttpServer {
|
||||||
|
|||||||
@@ -47,13 +47,13 @@
|
|||||||
//! # }
|
//! # }
|
||||||
//! ```
|
//! ```
|
||||||
|
|
||||||
pub mod traits;
|
|
||||||
pub mod job;
|
|
||||||
pub mod worker;
|
|
||||||
pub mod error;
|
pub mod error;
|
||||||
|
pub mod job;
|
||||||
#[cfg(feature = "in-memory")]
|
#[cfg(feature = "in-memory")]
|
||||||
pub mod in_memory;
|
pub mod in_memory;
|
||||||
|
pub mod traits;
|
||||||
|
pub mod worker;
|
||||||
|
|
||||||
#[cfg(feature = "in-memory")]
|
#[cfg(feature = "in-memory")]
|
||||||
pub use in_memory::InMemoryQueue;
|
pub use in_memory::InMemoryQueue;
|
||||||
|
|
||||||
@@ -61,3 +61,8 @@ pub use traits::Queue;
|
|||||||
pub use job::{Job, JobId};
|
pub use job::{Job, JobId};
|
||||||
pub use worker::{WorkerPool, WorkerConfig, JobHandler, JobFuture};
|
pub use worker::{WorkerPool, WorkerConfig, JobHandler, JobFuture};
|
||||||
pub use error::{QueueError, JobError};
|
pub use error::{QueueError, JobError};
|
||||||
|
|
||||||
|
#[cfg(feature = "in-memory")]
|
||||||
|
pub mod pipeline;
|
||||||
|
#[cfg(feature = "in-memory")]
|
||||||
|
pub use pipeline::{StageRunner, Stage, StageError};
|
||||||
|
|||||||
@@ -0,0 +1,137 @@
|
|||||||
|
//! Streaming pipeline that chains async transform stages with backpressure.
|
||||||
|
//!
|
||||||
|
//! Each stage processes items from the previous stage via a bounded channel,
|
||||||
|
//! providing natural backpressure between stages.
|
||||||
|
|
||||||
|
use std::marker::PhantomData;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
|
/// A single transform in the pipeline.
|
||||||
|
#[async_trait]
|
||||||
|
pub trait Stage<I: Send + 'static, O: Send + 'static>: Send + Sync {
|
||||||
|
async fn process(&self, input: I) -> Result<O, StageError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Errors from pipeline stages.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum StageError {
|
||||||
|
Processing(String),
|
||||||
|
ChannelClosed,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for StageError {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
StageError::Processing(s) => write!(f, "stage error: {s}"),
|
||||||
|
StageError::ChannelClosed => write!(f, "channel closed"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for StageError {}
|
||||||
|
|
||||||
|
/// Runs a single stage as a background task, consuming from input and
|
||||||
|
/// forwarding results to output.
|
||||||
|
pub struct StageRunner<S, I, O>
|
||||||
|
where
|
||||||
|
S: Stage<I, O>,
|
||||||
|
I: Send + 'static,
|
||||||
|
O: Send + 'static,
|
||||||
|
{
|
||||||
|
stage: Arc<S>,
|
||||||
|
_phantom: PhantomData<(I, O)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<S, I, O> StageRunner<S, I, O>
|
||||||
|
where
|
||||||
|
S: Stage<I, O> + 'static,
|
||||||
|
I: Send + 'static,
|
||||||
|
O: Send + 'static,
|
||||||
|
{
|
||||||
|
/// Creates a runner for a single stage with the given channel capacity.
|
||||||
|
pub fn new(stage: S) -> Self {
|
||||||
|
Self {
|
||||||
|
stage: Arc::new(stage),
|
||||||
|
_phantom: PhantomData,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Consumes items from `input`, applies the stage, sends to `output`.
|
||||||
|
/// Completes when the input stream ends.
|
||||||
|
pub fn run(
|
||||||
|
self,
|
||||||
|
input: mpsc::Receiver<I>,
|
||||||
|
output: mpsc::Sender<O>,
|
||||||
|
) -> JoinHandle<Result<(), StageError>>
|
||||||
|
where
|
||||||
|
S: 'static,
|
||||||
|
{
|
||||||
|
let stage = self.stage;
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut input = input;
|
||||||
|
loop {
|
||||||
|
match input.recv().await {
|
||||||
|
Some(item) => {
|
||||||
|
match stage.process(item).await {
|
||||||
|
Ok(out) => {
|
||||||
|
if output.send(out).await.is_err() {
|
||||||
|
return Err(StageError::ChannelClosed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => return Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<S, I, O> Default for StageRunner<S, I, O>
|
||||||
|
where
|
||||||
|
S: Stage<I, O> + Default + 'static,
|
||||||
|
I: Send + 'static,
|
||||||
|
O: Send + 'static,
|
||||||
|
{
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new(S::default())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
struct DoubleStage;
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Stage<u32, u32> for DoubleStage {
|
||||||
|
async fn process(&self, input: u32) -> Result<u32, StageError> {
|
||||||
|
Ok(input * 2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stage_runner_doubles_values() {
|
||||||
|
let (tx, rx_in) = mpsc::channel::<u32>(16);
|
||||||
|
let (tx_out, mut rx_out) = mpsc::channel::<u32>(16);
|
||||||
|
|
||||||
|
let runner = StageRunner::new(DoubleStage);
|
||||||
|
let handle = runner.run(rx_in, tx_out);
|
||||||
|
|
||||||
|
tx.send(5).await.unwrap();
|
||||||
|
tx.send(7).await.unwrap();
|
||||||
|
drop(tx);
|
||||||
|
|
||||||
|
assert_eq!(rx_out.recv().await, Some(10));
|
||||||
|
assert_eq!(rx_out.recv().await, Some(14));
|
||||||
|
assert_eq!(rx_out.recv().await, None);
|
||||||
|
|
||||||
|
assert!(handle.await.unwrap().is_ok());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,7 +13,7 @@ use crate::error::{QueueError, JobError};
|
|||||||
/// count). `ack`/`nack` are only valid on backends that support explicit
|
/// count). `ack`/`nack` are only valid on backends that support explicit
|
||||||
/// acknowledgment (NATS, Redis BLPOP-with-confirm). For in-memory and Postgres
|
/// acknowledgment (NATS, Redis BLPOP-with-confirm). For in-memory and Postgres
|
||||||
/// backends, the worker auto-acknowledges on `Ok` and auto-requeues on `Err`.
|
/// backends, the worker auto-acknowledges on `Ok` and auto-requeues on `Err`.
|
||||||
|
///
|
||||||
/// A trait for enqueueing and dequeueing jobs.
|
/// A trait for enqueueing and dequeueing jobs.
|
||||||
///
|
///
|
||||||
/// Implementations must be `Send + Sync`. Each backend provides its own factory
|
/// Implementations must be `Send + Sync`. Each backend provides its own factory
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ use std::pin::Pin;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use tokio::sync::Semaphore;
|
use tokio::sync::Semaphore;
|
||||||
|
|
||||||
use crate::error::JobError;
|
use crate::error::JobError;
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
//! Multipart upload trait for large-object uploads in parallel parts.
|
//! Multipart upload trait for large-object uploads in parallel parts.
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use std::pin::Pin;
|
|
||||||
use tokio::io::AsyncRead;
|
|
||||||
|
|
||||||
use crate::ObjectStream;
|
use crate::ObjectStream;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
//! Distributed lock with lease-based expiration.
|
||||||
|
//!
|
||||||
|
//! Provides a `DistributedLock` trait with in-process and Redis backends.
|
||||||
|
//! Used to coordinate leader election and queue dispatch across multiple
|
||||||
|
//! worker instances.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
|
/// Errors returned by distributed lock operations.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum LockError {
|
||||||
|
/// The lock could not be acquired (already held or timed out).
|
||||||
|
AlreadyHeld,
|
||||||
|
/// The lease expired and the lock was released.
|
||||||
|
Expired,
|
||||||
|
/// A backend transport error occurred.
|
||||||
|
Io(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for LockError {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
LockError::AlreadyHeld => write!(f, "lock already held"),
|
||||||
|
LockError::Expired => write!(f, "lock lease expired"),
|
||||||
|
LockError::Io(s) => write!(f, "lock io error: {s}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for LockError {}
|
||||||
|
|
||||||
|
/// Shared state for an in-process lock entry — maps keys to expiry instants.
|
||||||
|
type LockMap = Arc<Mutex<std::collections::HashMap<String, Instant>>>;
|
||||||
|
|
||||||
|
/// A handle to an acquired distributed lock (RAII — releases on drop).
|
||||||
|
pub struct LockGuard {
|
||||||
|
map: LockMap,
|
||||||
|
key: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LockGuard {
|
||||||
|
pub fn key(&self) -> &str {
|
||||||
|
&self.key
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Attempts to extend the lease.
|
||||||
|
pub async fn extend(&mut self, _dur: Duration) -> Result<(), LockError> {
|
||||||
|
Err(LockError::Expired)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for LockGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let map = Arc::clone(&self.map);
|
||||||
|
let key = self.key.clone();
|
||||||
|
// Fire-and-forget release: spawn a detached task to remove the key.
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut m = map.lock().await;
|
||||||
|
m.remove(&key);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Trait for distributed lock backends.
|
||||||
|
#[async_trait]
|
||||||
|
pub trait DistributedLock: Send + Sync {
|
||||||
|
/// Attempts to acquire the lock with the given lease duration.
|
||||||
|
async fn acquire(&self, key: &str, lease: Duration, timeout: Duration) -> Result<LockGuard, LockError>;
|
||||||
|
|
||||||
|
/// Releases the lock.
|
||||||
|
async fn release(&self, key: &str) -> Result<(), LockError>;
|
||||||
|
|
||||||
|
/// Attempts to extend an existing lease.
|
||||||
|
async fn extend(&self, key: &str, lease: Duration) -> Result<(), LockError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// In-process distributed lock using a mutex + lease timer.
|
||||||
|
/// Suitable for testing and single-instance coordination.
|
||||||
|
pub struct InProcLock {
|
||||||
|
held: LockMap,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl InProcLock {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
held: Arc::new(Mutex::new(std::collections::HashMap::new())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_expired(map: &std::collections::HashMap<String, Instant>, key: &str) -> bool {
|
||||||
|
if let Some(expiry) = map.get(key) {
|
||||||
|
*expiry <= Instant::now()
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for InProcLock {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl DistributedLock for InProcLock {
|
||||||
|
async fn acquire(&self, key: &str, lease: Duration, timeout: Duration) -> Result<LockGuard, LockError> {
|
||||||
|
let deadline = Instant::now() + timeout;
|
||||||
|
loop {
|
||||||
|
{
|
||||||
|
let mut map = self.held.lock().await;
|
||||||
|
// Clean up expired entries lazily.
|
||||||
|
map.retain(|_, v| *v > Instant::now());
|
||||||
|
if !map.contains_key(key) {
|
||||||
|
map.insert(key.to_string(), Instant::now() + lease);
|
||||||
|
return Ok(LockGuard {
|
||||||
|
map: Arc::clone(&self.held),
|
||||||
|
key: key.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if Instant::now() >= deadline {
|
||||||
|
return Err(LockError::AlreadyHeld);
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn release(&self, key: &str) -> Result<(), LockError> {
|
||||||
|
let mut map = self.held.lock().await;
|
||||||
|
map.remove(key);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn extend(&self, key: &str, lease: Duration) -> Result<(), LockError> {
|
||||||
|
let mut map = self.held.lock().await;
|
||||||
|
if let Some(entry) = map.get_mut(key) {
|
||||||
|
*entry = Instant::now() + lease;
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(LockError::AlreadyHeld)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn lock_acquire_release() {
|
||||||
|
let lock = InProcLock::new();
|
||||||
|
let guard = lock.acquire("key", Duration::from_secs(10), Duration::from_secs(1)).await.unwrap();
|
||||||
|
assert!(lock.release("key").await.is_ok());
|
||||||
|
drop(guard);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn lock_rejects_second_acquire() {
|
||||||
|
let lock = InProcLock::new();
|
||||||
|
let _guard1 = lock.acquire("key", Duration::from_secs(10), Duration::from_secs(1)).await.unwrap();
|
||||||
|
// While guard1 is alive, a second acquire with short timeout should fail.
|
||||||
|
let result = lock.acquire("key", Duration::from_secs(10), Duration::from_millis(50)).await;
|
||||||
|
assert!(result.is_err());
|
||||||
|
drop(_guard1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn lock_auto_releases_on_drop() {
|
||||||
|
let lock = InProcLock::new();
|
||||||
|
let guard = lock.acquire("k", Duration::from_secs(10), Duration::from_secs(1)).await.unwrap();
|
||||||
|
drop(guard);
|
||||||
|
// After drop, the lock should be releasable / re-acquirable.
|
||||||
|
let result = lock.acquire("k", Duration::from_secs(10), Duration::from_millis(50)).await;
|
||||||
|
assert!(result.is_ok(), "lock should be free after guard drop");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn lock_expires_after_lease() {
|
||||||
|
let lock = InProcLock::new();
|
||||||
|
let _guard = lock.acquire("key", Duration::from_millis(20), Duration::from_millis(5)).await.unwrap();
|
||||||
|
drop(_guard);
|
||||||
|
tokio::time::sleep(Duration::from_millis(30)).await;
|
||||||
|
// Should be acquirable now.
|
||||||
|
let result = lock.acquire("key", Duration::from_millis(20), Duration::from_millis(5)).await;
|
||||||
|
assert!(result.is_ok());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,6 @@
|
|||||||
//! Distributed leader election via Redis or in-process fallback.
|
//! Distributed leader election via Redis or in-process fallback.
|
||||||
|
|
||||||
use std::pin::Pin;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use tokio::sync::Notify;
|
use tokio::sync::Notify;
|
||||||
|
|||||||
@@ -61,6 +61,11 @@ pub mod metrics;
|
|||||||
#[cfg(feature = "observability")]
|
#[cfg(feature = "observability")]
|
||||||
pub mod panic_tracker;
|
pub mod panic_tracker;
|
||||||
|
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub mod service_builder;
|
||||||
|
#[cfg(feature = "lifecycle")]
|
||||||
|
pub mod dlock;
|
||||||
|
|
||||||
pub use context::{context, EngineContext};
|
pub use context::{context, EngineContext};
|
||||||
pub use error::MytheclipseError;
|
pub use error::MytheclipseError;
|
||||||
|
|
||||||
@@ -96,6 +101,12 @@ pub use health::{HealthCheck, HealthRegistry, HealthStatus};
|
|||||||
#[cfg(feature = "lifecycle")]
|
#[cfg(feature = "lifecycle")]
|
||||||
pub use leader::{InProcLeaderElection, LeaderElection};
|
pub use leader::{InProcLeaderElection, LeaderElection};
|
||||||
|
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub use service_builder::ServiceBuilder;
|
||||||
|
|
||||||
|
#[cfg(feature = "lifecycle")]
|
||||||
|
pub use dlock::{DistributedLock, LockError, LockGuard, InProcLock};
|
||||||
|
|
||||||
#[cfg(feature = "observability")]
|
#[cfg(feature = "observability")]
|
||||||
pub use metrics::{MetricsCollector, MetricsSnapshot};
|
pub use metrics::{MetricsCollector, MetricsSnapshot};
|
||||||
#[cfg(feature = "observability")]
|
#[cfg(feature = "observability")]
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
//! Provides a `Pool` trait and a built-in `SemaphorePool<T>` implementation
|
//! Provides a `Pool` trait and a built-in `SemaphorePool<T>` implementation
|
||||||
//! that distributes items drawn from a `Vec<T>` under a counting semaphore.
|
//! that distributes items drawn from a `Vec<T>` under a counting semaphore.
|
||||||
|
|
||||||
use std::pin::Pin;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|||||||
@@ -137,7 +137,7 @@ where
|
|||||||
/// Computes the (jittered) delay to sleep before retry `attempt` (1-based).
|
/// Computes the (jittered) delay to sleep before retry `attempt` (1-based).
|
||||||
///
|
///
|
||||||
/// Kept as a pure function for testability.
|
/// Kept as a pure function for testability.
|
||||||
fn backoff_delay<R: Rng>(config: &RetryConfig, attempt: u32, mut rng: R) -> Duration {
|
pub(crate) fn backoff_delay<R: Rng>(config: &RetryConfig, attempt: u32, mut rng: R) -> Duration {
|
||||||
let exponent = attempt.saturating_sub(1) as f64; // first retry uses base
|
let exponent = attempt.saturating_sub(1) as f64; // first retry uses base
|
||||||
let computed = config.base_delay.as_millis() as f64 * config.factor.powf(exponent);
|
let computed = config.base_delay.as_millis() as f64 * config.factor.powf(exponent);
|
||||||
let max_ms = config.max_delay.as_millis() as f64;
|
let max_ms = config.max_delay.as_millis() as f64;
|
||||||
|
|||||||
@@ -0,0 +1,291 @@
|
|||||||
|
//! Service builder that composes resiliency primitives.
|
||||||
|
//!
|
||||||
|
//! Provides `ServiceBuilder` for composing retry, circuit breaker, timeout,
|
||||||
|
//! and rate limiting around async service calls.
|
||||||
|
|
||||||
|
use std::future::Future;
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use tracing::Instrument;
|
||||||
|
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
use crate::circuit_breaker::CircuitBreaker;
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
use crate::retry::{retry, RetryConfig, RetryError};
|
||||||
|
#[cfg(feature = "traffic")]
|
||||||
|
use crate::ratelimit::RateLimiter;
|
||||||
|
|
||||||
|
/// Error returned by [`ServiceBuilder::run`].
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum RunError<E> {
|
||||||
|
Inner(E),
|
||||||
|
Retry(RetryError<E>),
|
||||||
|
CircuitOpen,
|
||||||
|
Timeout,
|
||||||
|
RateLimited,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<E: std::fmt::Display> std::fmt::Display for RunError<E> {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::Inner(e) => write!(f, "service failed: {e}"),
|
||||||
|
Self::Retry(e) => write!(f, "retry exhausted: {e}"),
|
||||||
|
Self::CircuitOpen => write!(f, "circuit breaker open"),
|
||||||
|
Self::Timeout => write!(f, "operation timed out"),
|
||||||
|
Self::RateLimited => write!(f, "rate limited"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
impl<E: std::fmt::Debug + std::fmt::Display + std::error::Error> std::error::Error for RunError<E> {}
|
||||||
|
#[cfg(not(feature = "resiliency"))]
|
||||||
|
impl<E: std::fmt::Debug + std::fmt::Display> std::error::Error for RunError<E> {}
|
||||||
|
|
||||||
|
// Config ------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[cfg(not(feature = "traffic"))]
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct ServiceConfig {
|
||||||
|
pub max_attempts: u32,
|
||||||
|
pub timeout: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(feature = "traffic"))]
|
||||||
|
impl Default for ServiceConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self { max_attempts: 0, timeout: Duration::ZERO }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "traffic")]
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct ServiceConfig {
|
||||||
|
pub max_attempts: u32,
|
||||||
|
pub timeout: Duration,
|
||||||
|
pub rate_per_sec: f64,
|
||||||
|
pub rate_burst: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "traffic")]
|
||||||
|
impl Default for ServiceConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self { max_attempts: 0, timeout: Duration::ZERO, rate_per_sec: 0.0, rate_burst: 0 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Builder -----------------------------------------------------------------
|
||||||
|
|
||||||
|
pub struct ServiceBuilder {
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
retry_cfg: Option<RetryConfig>,
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
circuit: Option<CircuitBreaker>,
|
||||||
|
#[cfg(feature = "traffic")]
|
||||||
|
rate_limiter: Option<RateLimiter>,
|
||||||
|
timeout: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ServiceBuilder {
|
||||||
|
pub fn new(config: ServiceConfig) -> Self {
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
let retry_cfg = (config.max_attempts > 0).then(|| RetryConfig {
|
||||||
|
max_attempts: config.max_attempts,
|
||||||
|
..RetryConfig::default()
|
||||||
|
});
|
||||||
|
|
||||||
|
#[cfg(feature = "traffic")]
|
||||||
|
let rate_limiter = {
|
||||||
|
if config.rate_per_sec > 0.0 && config.rate_burst > 0 {
|
||||||
|
Some(RateLimiter::new(config.rate_per_sec, config.rate_burst))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
Self {
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
retry_cfg,
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
circuit: None,
|
||||||
|
#[cfg(feature = "traffic")]
|
||||||
|
rate_limiter,
|
||||||
|
timeout: config.timeout,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub fn with_circuit_breaker(mut self, cb: CircuitBreaker) -> Self {
|
||||||
|
self.circuit = Some(cb);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "traffic")]
|
||||||
|
pub fn with_rate_limiter(mut self, rl: RateLimiter) -> Self {
|
||||||
|
self.rate_limiter = Some(rl);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_pre<E>(&self) -> Result<(), RunError<E>> {
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
if let Some(cb) = &self.circuit {
|
||||||
|
if !cb.allow_request() {
|
||||||
|
return Err(RunError::CircuitOpen);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[cfg(feature = "traffic")]
|
||||||
|
if let Some(rl) = &self.rate_limiter {
|
||||||
|
if rl.try_acquire().is_err() {
|
||||||
|
return Err(RunError::RateLimited);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
fn record(&self, ok: bool) {
|
||||||
|
if let Some(cb) = &self.circuit {
|
||||||
|
if ok { cb.record_success(); } else { cb.record_failure(); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn run<F, T, E>(&self, f: F) -> Result<T, RunError<E>>
|
||||||
|
where
|
||||||
|
F: FnMut() -> Pin<Box<dyn Future<Output = Result<T, E>> + Send>>,
|
||||||
|
E: std::fmt::Debug,
|
||||||
|
{
|
||||||
|
self.check_pre()?;
|
||||||
|
|
||||||
|
let dur = self.timeout;
|
||||||
|
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
{
|
||||||
|
if let Some(retry_cfg) = &self.retry_cfg {
|
||||||
|
let mut op = f;
|
||||||
|
let result: Result<T, RunError<E>> = if dur > Duration::ZERO {
|
||||||
|
// We can't easily combine retry + timeout with FnMut due to
|
||||||
|
// closure capture rules, so use a manual retry loop instead:
|
||||||
|
let cfg = retry_cfg.clone();
|
||||||
|
let mut attempt_no: u32 = 0;
|
||||||
|
let mut op_ref = op;
|
||||||
|
loop {
|
||||||
|
attempt_no += 1;
|
||||||
|
let span = tracing::info_span!("mytheclipse_service_call", attempt = attempt_no);
|
||||||
|
let fut = op_ref();
|
||||||
|
let attempt_result = tokio::time::timeout(dur, fut.instrument(span)).await;
|
||||||
|
match attempt_result {
|
||||||
|
Ok(Ok(v)) => {
|
||||||
|
self.record(true);
|
||||||
|
return Ok(v);
|
||||||
|
}
|
||||||
|
Ok(Err(e)) => {
|
||||||
|
self.record(false);
|
||||||
|
if attempt_no >= cfg.max_attempts {
|
||||||
|
return Err(RunError::Inner(e));
|
||||||
|
}
|
||||||
|
// retryable — backoff and retry
|
||||||
|
let delay = crate::retry::backoff_delay(&cfg, attempt_no, rand::thread_rng());
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
self.record(false);
|
||||||
|
if attempt_no >= cfg.max_attempts {
|
||||||
|
return Err(RunError::Timeout);
|
||||||
|
}
|
||||||
|
// retryable timeout — backoff and retry
|
||||||
|
let delay = crate::retry::backoff_delay(&cfg, attempt_no, rand::thread_rng());
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// retry() expects FnMut() -> Fut (not boxed), so adapt.
|
||||||
|
let mut inner_op = op;
|
||||||
|
retry(retry_cfg.clone(), |_: &E| true, || {
|
||||||
|
let span = tracing::info_span!("mytheclipse_service_call");
|
||||||
|
let fut = inner_op();
|
||||||
|
async move {
|
||||||
|
fut.instrument(span).await
|
||||||
|
}
|
||||||
|
}).await
|
||||||
|
.map_err(|e| {
|
||||||
|
self.record(false);
|
||||||
|
RunError::Retry(e)
|
||||||
|
})
|
||||||
|
.map(|v| {
|
||||||
|
self.record(true);
|
||||||
|
v
|
||||||
|
})
|
||||||
|
};
|
||||||
|
result
|
||||||
|
} else {
|
||||||
|
// No retry: just timeout or plain
|
||||||
|
let mut op = f;
|
||||||
|
let span = tracing::info_span!("mytheclipse_service_call");
|
||||||
|
let result = if dur > Duration::ZERO {
|
||||||
|
tokio::time::timeout(dur, op().instrument(span)).await
|
||||||
|
.map_err(|_| RunError::Timeout)?
|
||||||
|
.map_err(RunError::Inner)
|
||||||
|
} else {
|
||||||
|
op().instrument(span).await.map_err(RunError::Inner)
|
||||||
|
};
|
||||||
|
match &result {
|
||||||
|
Ok(_) => self.record(true),
|
||||||
|
Err(_) => self.record(false),
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(feature = "resiliency"))]
|
||||||
|
{
|
||||||
|
let _ = dur;
|
||||||
|
let mut op = f;
|
||||||
|
let span = tracing::info_span!("mytheclipse_service_call");
|
||||||
|
op().instrument(span).await.map_err(RunError::Inner)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn no_layers_passes_through() {
|
||||||
|
let builder = ServiceBuilder::new(ServiceConfig::default());
|
||||||
|
let result = builder.run(|| Box::pin(async { Ok::<_, ()>(42u32) })).await;
|
||||||
|
assert_eq!(result.unwrap(), 42);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn retry_succeeds_after_transient_failure() {
|
||||||
|
let mut cfg = ServiceConfig::default();
|
||||||
|
cfg.max_attempts = 3;
|
||||||
|
let builder = ServiceBuilder::new(cfg);
|
||||||
|
let attempts = Arc::new(std::sync::atomic::AtomicU32::new(0));
|
||||||
|
let result = builder.run(|| {
|
||||||
|
let a = Arc::clone(&attempts);
|
||||||
|
Box::pin(async move {
|
||||||
|
let n = a.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
if n < 2 { Err::<u32, _>(()) } else { Ok::<u32, _>(42) }
|
||||||
|
})
|
||||||
|
}).await;
|
||||||
|
assert_eq!(result.unwrap(), 42);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn timeout_returns_timeout_error() {
|
||||||
|
let mut cfg = ServiceConfig::default();
|
||||||
|
cfg.timeout = Duration::from_millis(5);
|
||||||
|
let builder = ServiceBuilder::new(cfg);
|
||||||
|
let result = builder.run(|| Box::pin(async {
|
||||||
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||||
|
Ok::<_, ()>(42u32)
|
||||||
|
})).await;
|
||||||
|
assert!(matches!(result, Err(RunError::Timeout)));
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user