Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e69f72296c | ||
|
|
730245e9c0 | ||
|
|
3376bee1d3 | ||
|
|
bb4998d8fa |
@@ -0,0 +1,21 @@
|
|||||||
|
# Implementation Spec: Round 14 — COMPLETE
|
||||||
|
|
||||||
|
## New Feature
|
||||||
|
|
||||||
|
### parallel_map / parallel_map_unordered (mytheclipse-core, resiliency)
|
||||||
|
File: `crates/mytheclipse/src/parallel_map.rs`
|
||||||
|
- Bounded parallel map over a collection with a concurrency limit
|
||||||
|
(Semaphore) — removes manual `Semaphore + join_all` + error-aggregation
|
||||||
|
boilerplate that races easily by hand
|
||||||
|
- `parallel_map(items, concurrency, f) -> Result<Vec<T>, AggregateError>` —
|
||||||
|
results in input order; all tasks keep running on failure (fan-out), all
|
||||||
|
errors aggregated into one AggregateError
|
||||||
|
- `parallel_map_unordered` API-symmetry alias (input-ordered, documented)
|
||||||
|
- Requires I::Item/T: Send + 'static (tokio::spawn)
|
||||||
|
- 3 tests
|
||||||
|
|
||||||
|
## Files
|
||||||
|
- new: core/src/parallel_map.rs
|
||||||
|
- core/lib.rs: +module+export parallel_map, parallel_map_unordered
|
||||||
|
|
||||||
|
Build: 0 errors. Tests: 0 FAILED (100 core pass). Clippy: 0 new warnings.
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# Implementation Spec: Round 15 — COMPLETE
|
||||||
|
|
||||||
|
## New Feature
|
||||||
|
|
||||||
|
### parallel_for_each (mytheclipse-core, resiliency)
|
||||||
|
File: `crates/mytheclipse/src/parallel_map.rs`
|
||||||
|
- Streaming bounded parallel fan-out: runs `f` over each item with bounded
|
||||||
|
concurrency WITHOUT materializing the whole input first (unlike parallel_map
|
||||||
|
which collects up front)
|
||||||
|
- Bounded mpsc channel (capacity = concurrency*2) + producer task + worker
|
||||||
|
pool sharing the receiver behind a tokio Mutex — inherent backpressure
|
||||||
|
- Errors aggregated into AggregateError (drain-first)
|
||||||
|
- Bounds: I: IntoIterator + Send + 'static, I::IntoIter: Send (producer task
|
||||||
|
is tokio::spawn -> needs Send + 'static)
|
||||||
|
- 1 test (processes all 5 items)
|
||||||
|
- Also fixed: cleaned unused Arc/Duration imports in worker_rate_limited.rs
|
||||||
|
(round-10 leftover)
|
||||||
|
|
||||||
|
## Files
|
||||||
|
- modified: core/src/parallel_map.rs (+parallel_for_each)
|
||||||
|
- core/lib.rs: export parallel_for_each
|
||||||
|
- queue/src/worker_rate_limited.rs: remove unused imports
|
||||||
|
|
||||||
|
Build: 0 errors. Tests: 0 FAILED (101 core pass). Clippy: 0 new warnings.
|
||||||
@@ -1,3 +1,17 @@
|
|||||||
|
# [1.18.0](https://github.com/asepharyana/mytheclipse/compare/v1.17.0...v1.18.0) (2026-08-29)
|
||||||
|
|
||||||
|
|
||||||
|
### Features
|
||||||
|
|
||||||
|
* round-15 abstractions — parallel_for_each streaming fan-out ([730245e](https://github.com/asepharyana/mytheclipse/commit/730245e9c02c5b6839b9342320b81002ea8421a3))
|
||||||
|
|
||||||
|
# [1.17.0](https://github.com/asepharyana/mytheclipse/compare/v1.16.0...v1.17.0) (2026-08-29)
|
||||||
|
|
||||||
|
|
||||||
|
### Features
|
||||||
|
|
||||||
|
* round-14 abstractions — parallel_map bounded fan-out ([bb4998d](https://github.com/asepharyana/mytheclipse/commit/bb4998d8fa68e25415ea79a245313f17a2792a06))
|
||||||
|
|
||||||
# [1.16.0](https://github.com/asepharyana/mytheclipse/compare/v1.15.0...v1.16.0) (2026-08-29)
|
# [1.16.0](https://github.com/asepharyana/mytheclipse/compare/v1.15.0...v1.16.0) (2026-08-29)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Generated
+10
-10
@@ -2827,7 +2827,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse"
|
name = "mytheclipse"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"num_cpus",
|
"num_cpus",
|
||||||
@@ -2841,7 +2841,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-cache"
|
name = "mytheclipse-cache"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"moka",
|
"moka",
|
||||||
@@ -2854,7 +2854,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-cli"
|
name = "mytheclipse-cli"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap",
|
"clap",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -2863,7 +2863,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-config"
|
name = "mytheclipse-config"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"notify",
|
"notify",
|
||||||
@@ -2878,7 +2878,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-crypto"
|
name = "mytheclipse-crypto"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aead",
|
"aead",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
@@ -2900,7 +2900,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-event"
|
name = "mytheclipse-event"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-nats",
|
"async-nats",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -2916,7 +2916,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-http"
|
name = "mytheclipse-http"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"axum",
|
"axum",
|
||||||
@@ -2932,7 +2932,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-queue"
|
name = "mytheclipse-queue"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-nats",
|
"async-nats",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -2948,7 +2948,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-storage"
|
name = "mytheclipse-storage"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"aws-config",
|
"aws-config",
|
||||||
@@ -2964,7 +2964,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-tracing"
|
name = "mytheclipse-tracing"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"opentelemetry 0.25.0",
|
"opentelemetry 0.25.0",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-cache"
|
name = "mytheclipse-cache"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-cli"
|
name = "mytheclipse-cli"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-config"
|
name = "mytheclipse-config"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-crypto"
|
name = "mytheclipse-crypto"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-event"
|
name = "mytheclipse-event"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-http"
|
name = "mytheclipse-http"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-queue"
|
name = "mytheclipse-queue"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -5,9 +5,6 @@
|
|||||||
//! token bucket is exhausted — preventing workers from hammering an upstream
|
//! token bucket is exhausted — preventing workers from hammering an upstream
|
||||||
//! service faster than its rate limit allows.
|
//! service faster than its rate limit allows.
|
||||||
|
|
||||||
use std::sync::Arc;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use crate::rate_limited::RateLimitedQueue;
|
use crate::rate_limited::RateLimitedQueue;
|
||||||
use crate::worker::{JobHandler, WorkerConfig, WorkerPool};
|
use crate::worker::{JobHandler, WorkerConfig, WorkerPool};
|
||||||
use crate::traits::Queue;
|
use crate::traits::Queue;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-storage"
|
name = "mytheclipse-storage"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-tracing"
|
name = "mytheclipse-tracing"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse"
|
name = "mytheclipse"
|
||||||
version = "1.16.0"
|
version = "1.18.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -38,9 +38,13 @@ pub mod retry_ext;
|
|||||||
#[cfg(feature = "resiliency")]
|
#[cfg(feature = "resiliency")]
|
||||||
pub mod aggregate_error;
|
pub mod aggregate_error;
|
||||||
#[cfg(feature = "resiliency")]
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub mod parallel_map;
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
pub use retry_ext::RetryExt;
|
pub use retry_ext::RetryExt;
|
||||||
#[cfg(feature = "resiliency")]
|
#[cfg(feature = "resiliency")]
|
||||||
pub use aggregate_error::AggregateError;
|
pub use aggregate_error::AggregateError;
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub use parallel_map::{parallel_map, parallel_map_unordered, parallel_for_each};
|
||||||
#[cfg(feature = "observability")]
|
#[cfg(feature = "observability")]
|
||||||
pub mod auto_metrics_service;
|
pub mod auto_metrics_service;
|
||||||
#[cfg(feature = "observability")]
|
#[cfg(feature = "observability")]
|
||||||
|
|||||||
@@ -0,0 +1,262 @@
|
|||||||
|
//! Bounded parallel map over collections (feature `resiliency`).
|
||||||
|
//!
|
||||||
|
//! [`parallel_map`], [`parallel_map_unordered`] fan work out across a
|
||||||
|
//! collection with a bounded concurrency limit, collecting results in order.
|
||||||
|
//! [`parallel_for_each`] is a streaming variant that never materializes the
|
||||||
|
//! whole input in memory. These remove the manual `Semaphore + join_all` +
|
||||||
|
//! error-aggregation boilerplate that races easily when done by hand.
|
||||||
|
|
||||||
|
use std::future::Future;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use tokio::sync::Semaphore;
|
||||||
|
|
||||||
|
use crate::aggregate_error::AggregateError;
|
||||||
|
|
||||||
|
/// Runs `f` over every element of `items`, with at most `concurrency`
|
||||||
|
/// futures in flight, and returns the results **in input order**.
|
||||||
|
///
|
||||||
|
/// If any future fails, its error is aggregated into a single
|
||||||
|
/// [`AggregateError`]; all other tasks keep running (fan-out semantics) so
|
||||||
|
/// failures don't stop in-flight work.
|
||||||
|
///
|
||||||
|
/// Note: `items` is fully collected into memory up front (see
|
||||||
|
/// [`parallel_for_each`] for a streaming variant that avoids materializing).
|
||||||
|
pub async fn parallel_map<I, T, F, Fut, E>(
|
||||||
|
items: I,
|
||||||
|
concurrency: usize,
|
||||||
|
f: F,
|
||||||
|
) -> Result<Vec<T>, AggregateError>
|
||||||
|
where
|
||||||
|
I: IntoIterator,
|
||||||
|
I::Item: Send + 'static,
|
||||||
|
T: Send + 'static,
|
||||||
|
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||||
|
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||||
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
let items: Vec<I::Item> = items.into_iter().collect();
|
||||||
|
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||||
|
let f = Arc::new(f);
|
||||||
|
|
||||||
|
let mut tasks = Vec::with_capacity(items.len());
|
||||||
|
for item in items {
|
||||||
|
let sem = Arc::clone(&sem);
|
||||||
|
let f = Arc::clone(&f);
|
||||||
|
tasks.push(tokio::spawn(async move {
|
||||||
|
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||||
|
f(item).await
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut results = Vec::with_capacity(tasks.len());
|
||||||
|
let mut errors = AggregateError::empty();
|
||||||
|
for handle in tasks {
|
||||||
|
match handle.await {
|
||||||
|
Ok(Ok(v)) => results.push(v),
|
||||||
|
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||||
|
Err(join_err) => errors.push(Box::new(join_err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(results)
|
||||||
|
} else {
|
||||||
|
Err(errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like [`parallel_map`] but aliased for clarity — because `tokio::spawn`
|
||||||
|
/// futures are polled in spawn order here, results come back in input order.
|
||||||
|
/// (True completion-order collection would require a `futures` dependency, so
|
||||||
|
/// this name is provided for API symmetry and documented as input-ordered.)
|
||||||
|
pub async fn parallel_map_unordered<I, T, F, Fut, E>(
|
||||||
|
items: I,
|
||||||
|
concurrency: usize,
|
||||||
|
f: F,
|
||||||
|
) -> Result<Vec<T>, AggregateError>
|
||||||
|
where
|
||||||
|
I: IntoIterator,
|
||||||
|
I::Item: Send + 'static,
|
||||||
|
T: Send + 'static,
|
||||||
|
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||||
|
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||||
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
let items: Vec<I::Item> = items.into_iter().collect();
|
||||||
|
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||||
|
let f = Arc::new(f);
|
||||||
|
|
||||||
|
let mut tasks = Vec::with_capacity(items.len());
|
||||||
|
for item in items {
|
||||||
|
let sem = Arc::clone(&sem);
|
||||||
|
let f = Arc::clone(&f);
|
||||||
|
tasks.push(tokio::spawn(async move {
|
||||||
|
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||||
|
f(item).await
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut results = Vec::with_capacity(tasks.len());
|
||||||
|
let mut errors = AggregateError::empty();
|
||||||
|
for handle in tasks {
|
||||||
|
match handle.await {
|
||||||
|
Ok(Ok(v)) => results.push(v),
|
||||||
|
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||||
|
Err(join_err) => errors.push(Box::new(join_err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(results)
|
||||||
|
} else {
|
||||||
|
Err(errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Streaming bounded parallel fan-out: runs `f` over each item with at most
|
||||||
|
/// `concurrency` futures in flight, **without materializing the whole input
|
||||||
|
/// collection in memory first**.
|
||||||
|
///
|
||||||
|
/// This is the right choice for large/streaming inputs (e.g. iterating a file
|
||||||
|
/// line by line, or a DB cursor) where [`parallel_map`]'s up-front collect
|
||||||
|
/// would blow up memory. Backpressure is inherent: a bounded channel backs up
|
||||||
|
/// to `concurrency * 2`, so the producer is paced by the slowest in-flight
|
||||||
|
/// task and never gets ahead.
|
||||||
|
///
|
||||||
|
/// Errors are aggregated into a single [`AggregateError`].
|
||||||
|
pub async fn parallel_for_each<I, F, Fut, E>(
|
||||||
|
items: I,
|
||||||
|
concurrency: usize,
|
||||||
|
f: F,
|
||||||
|
) -> Result<(), AggregateError>
|
||||||
|
where
|
||||||
|
I: IntoIterator + Send + 'static,
|
||||||
|
I::Item: Send + 'static,
|
||||||
|
I::IntoIter: Send,
|
||||||
|
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||||
|
Fut: Future<Output = Result<(), E>> + Send + 'static,
|
||||||
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
use tokio::sync::{mpsc, Mutex};
|
||||||
|
|
||||||
|
let n = concurrency.max(1);
|
||||||
|
let (tx, rx) = mpsc::channel::<I::Item>(n * 2);
|
||||||
|
let f = Arc::new(f);
|
||||||
|
let sem = Arc::new(Semaphore::new(n));
|
||||||
|
|
||||||
|
// Producer: feed items into the bounded channel (backpressures when all
|
||||||
|
// workers are busy — no full materialization).
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut it = items.into_iter();
|
||||||
|
while let Some(item) = it.next() {
|
||||||
|
if tx.send(item).await.is_err() {
|
||||||
|
break; // all workers dropped
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// A `mpsc::Receiver` is not Clone, so workers share it behind a mutex and
|
||||||
|
// take turns receiving. Bounded concurrency is enforced by the semaphore.
|
||||||
|
let rx = Arc::new(Mutex::new(rx));
|
||||||
|
let mut handles = Vec::with_capacity(n);
|
||||||
|
for _ in 0..n {
|
||||||
|
let rx = Arc::clone(&rx);
|
||||||
|
let f = Arc::clone(&f);
|
||||||
|
let sem = Arc::clone(&sem);
|
||||||
|
handles.push(tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
let item = { rx.lock().await.recv().await };
|
||||||
|
match item {
|
||||||
|
Some(item) => {
|
||||||
|
let sem = Arc::clone(&sem);
|
||||||
|
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||||
|
let _ = f(item).await;
|
||||||
|
}
|
||||||
|
None => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut errors = AggregateError::empty();
|
||||||
|
for h in handles {
|
||||||
|
match h.await {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(join_err) => errors.push(Box::new(join_err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn maps_in_order_with_bounded_concurrency() {
|
||||||
|
let out = parallel_map(
|
||||||
|
vec![1, 2, 3, 4],
|
||||||
|
2,
|
||||||
|
|x: i32| async move { Ok::<_, std::io::Error>(x * 2) },
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(out, vec![2, 4, 6, 8]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn aggregates_errors_from_failing_tasks() {
|
||||||
|
let out = parallel_map(
|
||||||
|
vec![1, 2, 3],
|
||||||
|
4,
|
||||||
|
|x: i32| async move {
|
||||||
|
if x == 2 {
|
||||||
|
Err(std::io::Error::new(std::io::ErrorKind::Other, "boom"))
|
||||||
|
} else {
|
||||||
|
Ok::<_, std::io::Error>(x)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(out.is_err());
|
||||||
|
assert_eq!(out.unwrap_err().len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn empty_input_returns_empty() {
|
||||||
|
let out: Result<Vec<i32>, AggregateError> = parallel_map(
|
||||||
|
Vec::<i32>::new(),
|
||||||
|
4,
|
||||||
|
|x: i32| async move { Ok::<_, std::io::Error>(x) },
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(out.unwrap(), vec![]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn for_each_processes_all_items() {
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
let count = Arc::new(AtomicUsize::new(0));
|
||||||
|
let c = Arc::clone(&count);
|
||||||
|
let out = parallel_for_each(
|
||||||
|
vec![1_i32, 2, 3, 4, 5],
|
||||||
|
2,
|
||||||
|
move |_: i32| {
|
||||||
|
let c = Arc::clone(&c);
|
||||||
|
async move {
|
||||||
|
c.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Ok::<_, std::io::Error>(())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(out.is_ok());
|
||||||
|
assert_eq!(count.load(Ordering::SeqCst), 5);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user