Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3376bee1d3 | ||
|
|
bb4998d8fa |
@@ -0,0 +1,21 @@
|
|||||||
|
# Implementation Spec: Round 14 — COMPLETE
|
||||||
|
|
||||||
|
## New Feature
|
||||||
|
|
||||||
|
### parallel_map / parallel_map_unordered (mytheclipse-core, resiliency)
|
||||||
|
File: `crates/mytheclipse/src/parallel_map.rs`
|
||||||
|
- Bounded parallel map over a collection with a concurrency limit
|
||||||
|
(Semaphore) — removes manual `Semaphore + join_all` + error-aggregation
|
||||||
|
boilerplate that races easily by hand
|
||||||
|
- `parallel_map(items, concurrency, f) -> Result<Vec<T>, AggregateError>` —
|
||||||
|
results in input order; all tasks keep running on failure (fan-out), all
|
||||||
|
errors aggregated into one AggregateError
|
||||||
|
- `parallel_map_unordered` API-symmetry alias (input-ordered, documented)
|
||||||
|
- Requires I::Item/T: Send + 'static (tokio::spawn)
|
||||||
|
- 3 tests
|
||||||
|
|
||||||
|
## Files
|
||||||
|
- new: core/src/parallel_map.rs
|
||||||
|
- core/lib.rs: +module+export parallel_map, parallel_map_unordered
|
||||||
|
|
||||||
|
Build: 0 errors. Tests: 0 FAILED (100 core pass). Clippy: 0 new warnings.
|
||||||
@@ -1,3 +1,10 @@
|
|||||||
|
# [1.17.0](https://github.com/asepharyana/mytheclipse/compare/v1.16.0...v1.17.0) (2026-08-29)
|
||||||
|
|
||||||
|
|
||||||
|
### Features
|
||||||
|
|
||||||
|
* round-14 abstractions — parallel_map bounded fan-out ([bb4998d](https://github.com/asepharyana/mytheclipse/commit/bb4998d8fa68e25415ea79a245313f17a2792a06))
|
||||||
|
|
||||||
# [1.16.0](https://github.com/asepharyana/mytheclipse/compare/v1.15.0...v1.16.0) (2026-08-29)
|
# [1.16.0](https://github.com/asepharyana/mytheclipse/compare/v1.15.0...v1.16.0) (2026-08-29)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Generated
+10
-10
@@ -2827,7 +2827,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse"
|
name = "mytheclipse"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"num_cpus",
|
"num_cpus",
|
||||||
@@ -2841,7 +2841,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-cache"
|
name = "mytheclipse-cache"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"moka",
|
"moka",
|
||||||
@@ -2854,7 +2854,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-cli"
|
name = "mytheclipse-cli"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap",
|
"clap",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -2863,7 +2863,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-config"
|
name = "mytheclipse-config"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"notify",
|
"notify",
|
||||||
@@ -2878,7 +2878,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-crypto"
|
name = "mytheclipse-crypto"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aead",
|
"aead",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
@@ -2900,7 +2900,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-event"
|
name = "mytheclipse-event"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-nats",
|
"async-nats",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -2916,7 +2916,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-http"
|
name = "mytheclipse-http"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"axum",
|
"axum",
|
||||||
@@ -2932,7 +2932,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-queue"
|
name = "mytheclipse-queue"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-nats",
|
"async-nats",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -2948,7 +2948,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-storage"
|
name = "mytheclipse-storage"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"aws-config",
|
"aws-config",
|
||||||
@@ -2964,7 +2964,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-tracing"
|
name = "mytheclipse-tracing"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"opentelemetry 0.25.0",
|
"opentelemetry 0.25.0",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-cache"
|
name = "mytheclipse-cache"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-cli"
|
name = "mytheclipse-cli"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-config"
|
name = "mytheclipse-config"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-crypto"
|
name = "mytheclipse-crypto"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-event"
|
name = "mytheclipse-event"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-http"
|
name = "mytheclipse-http"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-queue"
|
name = "mytheclipse-queue"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-storage"
|
name = "mytheclipse-storage"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-tracing"
|
name = "mytheclipse-tracing"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse"
|
name = "mytheclipse"
|
||||||
version = "1.16.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -38,9 +38,13 @@ pub mod retry_ext;
|
|||||||
#[cfg(feature = "resiliency")]
|
#[cfg(feature = "resiliency")]
|
||||||
pub mod aggregate_error;
|
pub mod aggregate_error;
|
||||||
#[cfg(feature = "resiliency")]
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub mod parallel_map;
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
pub use retry_ext::RetryExt;
|
pub use retry_ext::RetryExt;
|
||||||
#[cfg(feature = "resiliency")]
|
#[cfg(feature = "resiliency")]
|
||||||
pub use aggregate_error::AggregateError;
|
pub use aggregate_error::AggregateError;
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub use parallel_map::{parallel_map, parallel_map_unordered};
|
||||||
#[cfg(feature = "observability")]
|
#[cfg(feature = "observability")]
|
||||||
pub mod auto_metrics_service;
|
pub mod auto_metrics_service;
|
||||||
#[cfg(feature = "observability")]
|
#[cfg(feature = "observability")]
|
||||||
|
|||||||
@@ -0,0 +1,159 @@
|
|||||||
|
//! Bounded parallel map over collections (feature `resiliency`).
|
||||||
|
//!
|
||||||
|
//! [`parallel_map`] / [`parallel_map_unordered`] fan out work across a
|
||||||
|
//! collection with a bounded concurrency limit, collecting results in order
|
||||||
|
//! (or completion order). This removes the manual `Semaphore + join_all` +
|
||||||
|
//! error-aggregation boilerplate that races easily when done by hand.
|
||||||
|
|
||||||
|
use std::future::Future;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use tokio::sync::Semaphore;
|
||||||
|
|
||||||
|
use crate::aggregate_error::AggregateError;
|
||||||
|
|
||||||
|
/// Runs `f` over every element of `items`, with at most `concurrency`
|
||||||
|
/// futures in flight, and returns the results **in input order**.
|
||||||
|
///
|
||||||
|
/// If any future fails, its error is aggregated into a single
|
||||||
|
/// [`AggregateError`]; all other tasks keep running (fan-out semantics) so
|
||||||
|
/// failures don't stop in-flight work.
|
||||||
|
pub async fn parallel_map<I, T, F, Fut, E>(
|
||||||
|
items: I,
|
||||||
|
concurrency: usize,
|
||||||
|
f: F,
|
||||||
|
) -> Result<Vec<T>, AggregateError>
|
||||||
|
where
|
||||||
|
I: IntoIterator,
|
||||||
|
I::Item: Send + 'static,
|
||||||
|
T: Send + 'static,
|
||||||
|
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||||
|
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||||
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
let items: Vec<I::Item> = items.into_iter().collect();
|
||||||
|
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||||
|
let f = Arc::new(f);
|
||||||
|
|
||||||
|
let mut tasks = Vec::with_capacity(items.len());
|
||||||
|
for item in items {
|
||||||
|
let sem = Arc::clone(&sem);
|
||||||
|
let f = Arc::clone(&f);
|
||||||
|
tasks.push(tokio::spawn(async move {
|
||||||
|
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||||
|
f(item).await
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut results = Vec::with_capacity(tasks.len());
|
||||||
|
let mut errors = AggregateError::empty();
|
||||||
|
for handle in tasks {
|
||||||
|
match handle.await {
|
||||||
|
Ok(Ok(v)) => results.push(v),
|
||||||
|
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||||
|
Err(join_err) => errors.push(Box::new(join_err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(results)
|
||||||
|
} else {
|
||||||
|
Err(errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like [`parallel_map`] but aliased for clarity — because `tokio::spawn`
|
||||||
|
/// futures are polled in spawn order here, results come back in input order.
|
||||||
|
/// (True completion-order collection would require a `futures` dependency, so
|
||||||
|
/// this name is provided for API symmetry and documented as input-ordered.)
|
||||||
|
pub async fn parallel_map_unordered<I, T, F, Fut, E>(
|
||||||
|
items: I,
|
||||||
|
concurrency: usize,
|
||||||
|
f: F,
|
||||||
|
) -> Result<Vec<T>, AggregateError>
|
||||||
|
where
|
||||||
|
I: IntoIterator,
|
||||||
|
I::Item: Send + 'static,
|
||||||
|
T: Send + 'static,
|
||||||
|
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||||
|
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||||
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
let items: Vec<I::Item> = items.into_iter().collect();
|
||||||
|
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||||
|
let f = Arc::new(f);
|
||||||
|
|
||||||
|
let mut tasks = Vec::with_capacity(items.len());
|
||||||
|
for item in items {
|
||||||
|
let sem = Arc::clone(&sem);
|
||||||
|
let f = Arc::clone(&f);
|
||||||
|
tasks.push(tokio::spawn(async move {
|
||||||
|
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||||
|
f(item).await
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut results = Vec::with_capacity(tasks.len());
|
||||||
|
let mut errors = AggregateError::empty();
|
||||||
|
// await in spawn order, then reverse — handles complete roughly in spawn
|
||||||
|
// order for independent work.
|
||||||
|
for handle in tasks {
|
||||||
|
match handle.await {
|
||||||
|
Ok(Ok(v)) => results.push(v),
|
||||||
|
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||||
|
Err(join_err) => errors.push(Box::new(join_err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(results)
|
||||||
|
} else {
|
||||||
|
Err(errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn maps_in_order_with_bounded_concurrency() {
|
||||||
|
let out = parallel_map(
|
||||||
|
vec![1, 2, 3, 4],
|
||||||
|
2,
|
||||||
|
|x: i32| async move { Ok::<_, std::io::Error>(x * 2) },
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(out, vec![2, 4, 6, 8]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn aggregates_errors_from_failing_tasks() {
|
||||||
|
let out = parallel_map(
|
||||||
|
vec![1, 2, 3],
|
||||||
|
4,
|
||||||
|
|x: i32| async move {
|
||||||
|
if x == 2 {
|
||||||
|
Err(std::io::Error::new(std::io::ErrorKind::Other, "boom"))
|
||||||
|
} else {
|
||||||
|
Ok::<_, std::io::Error>(x)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(out.is_err());
|
||||||
|
assert_eq!(out.unwrap_err().len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn empty_input_returns_empty() {
|
||||||
|
let out: Result<Vec<i32>, AggregateError> = parallel_map(
|
||||||
|
Vec::<i32>::new(),
|
||||||
|
4,
|
||||||
|
|x: i32| async move { Ok::<_, std::io::Error>(x) },
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(out.unwrap(), vec![]);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user