Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3376bee1d3 | ||
|
|
bb4998d8fa | ||
|
|
125bc57290 | ||
|
|
ff5fbe49dc |
@@ -0,0 +1,19 @@
|
|||||||
|
# Implementation Spec: Round 13 — COMPLETE
|
||||||
|
|
||||||
|
## New Feature
|
||||||
|
|
||||||
|
### AggregateError (mytheclipse-core, resiliency)
|
||||||
|
File: `crates/mytheclipse/src/aggregate_error.rs`
|
||||||
|
- Collects multiple `E: std::error::Error` from parallel/fan-out tasks into one
|
||||||
|
error — natural failure type for `join_all` + batch/fan-out resilience
|
||||||
|
- `empty()` / `with_context(..)` / push(E) / is_empty / len / iter
|
||||||
|
- `from_results(Vec<Result<V,E>>) -> Result<Vec<V>, AggregateError>` — collects
|
||||||
|
ALL errors, returns values when all Ok
|
||||||
|
- Display lists count + first error; From<Vec<Box<dyn Error>>>, Extend
|
||||||
|
- 3 tests
|
||||||
|
|
||||||
|
## Files
|
||||||
|
- new: core/src/aggregate_error.rs
|
||||||
|
- core/lib.rs: +module+export AggregateError (resiliency)
|
||||||
|
|
||||||
|
Build: exit 0. Tests: 0 FAILED (97 core pass). Clippy: 0 new warnings.
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# Implementation Spec: Round 14 — COMPLETE
|
||||||
|
|
||||||
|
## New Feature
|
||||||
|
|
||||||
|
### parallel_map / parallel_map_unordered (mytheclipse-core, resiliency)
|
||||||
|
File: `crates/mytheclipse/src/parallel_map.rs`
|
||||||
|
- Bounded parallel map over a collection with a concurrency limit
|
||||||
|
(Semaphore) — removes manual `Semaphore + join_all` + error-aggregation
|
||||||
|
boilerplate that races easily by hand
|
||||||
|
- `parallel_map(items, concurrency, f) -> Result<Vec<T>, AggregateError>` —
|
||||||
|
results in input order; all tasks keep running on failure (fan-out), all
|
||||||
|
errors aggregated into one AggregateError
|
||||||
|
- `parallel_map_unordered` API-symmetry alias (input-ordered, documented)
|
||||||
|
- Requires I::Item/T: Send + 'static (tokio::spawn)
|
||||||
|
- 3 tests
|
||||||
|
|
||||||
|
## Files
|
||||||
|
- new: core/src/parallel_map.rs
|
||||||
|
- core/lib.rs: +module+export parallel_map, parallel_map_unordered
|
||||||
|
|
||||||
|
Build: 0 errors. Tests: 0 FAILED (100 core pass). Clippy: 0 new warnings.
|
||||||
@@ -1,3 +1,17 @@
|
|||||||
|
# [1.17.0](https://github.com/asepharyana/mytheclipse/compare/v1.16.0...v1.17.0) (2026-08-29)
|
||||||
|
|
||||||
|
|
||||||
|
### Features
|
||||||
|
|
||||||
|
* round-14 abstractions — parallel_map bounded fan-out ([bb4998d](https://github.com/asepharyana/mytheclipse/commit/bb4998d8fa68e25415ea79a245313f17a2792a06))
|
||||||
|
|
||||||
|
# [1.16.0](https://github.com/asepharyana/mytheclipse/compare/v1.15.0...v1.16.0) (2026-08-29)
|
||||||
|
|
||||||
|
|
||||||
|
### Features
|
||||||
|
|
||||||
|
* round-13 abstractions — AggregateError for parallel fan-out ([ff5fbe4](https://github.com/asepharyana/mytheclipse/commit/ff5fbe49dc1ff06c287306835110a6652d6b24b9))
|
||||||
|
|
||||||
# [1.15.0](https://github.com/asepharyana/mytheclipse/compare/v1.14.0...v1.15.0) (2026-08-29)
|
# [1.15.0](https://github.com/asepharyana/mytheclipse/compare/v1.14.0...v1.15.0) (2026-08-29)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Generated
+10
-10
@@ -2827,7 +2827,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse"
|
name = "mytheclipse"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"num_cpus",
|
"num_cpus",
|
||||||
@@ -2841,7 +2841,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-cache"
|
name = "mytheclipse-cache"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"moka",
|
"moka",
|
||||||
@@ -2854,7 +2854,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-cli"
|
name = "mytheclipse-cli"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap",
|
"clap",
|
||||||
"tokio",
|
"tokio",
|
||||||
@@ -2863,7 +2863,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-config"
|
name = "mytheclipse-config"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"dotenvy",
|
"dotenvy",
|
||||||
"notify",
|
"notify",
|
||||||
@@ -2878,7 +2878,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-crypto"
|
name = "mytheclipse-crypto"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aead",
|
"aead",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
@@ -2900,7 +2900,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-event"
|
name = "mytheclipse-event"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-nats",
|
"async-nats",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -2916,7 +2916,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-http"
|
name = "mytheclipse-http"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"axum",
|
"axum",
|
||||||
@@ -2932,7 +2932,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-queue"
|
name = "mytheclipse-queue"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-nats",
|
"async-nats",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -2948,7 +2948,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-storage"
|
name = "mytheclipse-storage"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"aws-config",
|
"aws-config",
|
||||||
@@ -2964,7 +2964,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mytheclipse-tracing"
|
name = "mytheclipse-tracing"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"opentelemetry 0.25.0",
|
"opentelemetry 0.25.0",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-cache"
|
name = "mytheclipse-cache"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-cli"
|
name = "mytheclipse-cli"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-config"
|
name = "mytheclipse-config"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-crypto"
|
name = "mytheclipse-crypto"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-event"
|
name = "mytheclipse-event"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-http"
|
name = "mytheclipse-http"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-queue"
|
name = "mytheclipse-queue"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-storage"
|
name = "mytheclipse-storage"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse-tracing"
|
name = "mytheclipse-tracing"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "mytheclipse"
|
name = "mytheclipse"
|
||||||
version = "1.15.0"
|
version = "1.17.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.75"
|
rust-version = "1.75"
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
//! Error aggregation for parallel/fan-out work (feature `resiliency`).
|
||||||
|
//!
|
||||||
|
//! [`AggregateError`] collects multiple `E: std::error::Error` values produced
|
||||||
|
//! by concurrently executed tasks into one error, so a caller awaiting `N`
|
||||||
|
//! tasks via `join_all` can surface *every* failure at once instead of
|
||||||
|
//! stopping at the first. This is the natural failure type for
|
||||||
|
//! `futures::future::join_all(vec![...])` transactions, batch operations, and
|
||||||
|
//! fan-out resilience.
|
||||||
|
|
||||||
|
use std::fmt;
|
||||||
|
|
||||||
|
/// An error that groups one or more underlying errors.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct AggregateError {
|
||||||
|
errors: Vec<Box<dyn std::error::Error + Send + Sync>>,
|
||||||
|
/// Optional label describing the operation that failed.
|
||||||
|
context: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AggregateError {
|
||||||
|
/// Creates an empty aggregate (no errors yet).
|
||||||
|
pub fn empty() -> Self {
|
||||||
|
Self {
|
||||||
|
errors: Vec::new(),
|
||||||
|
context: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Creates a labeled aggregate with an operation context.
|
||||||
|
pub fn with_context(context: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
errors: Vec::new(),
|
||||||
|
context: Some(context.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Adds an error to the aggregate.
|
||||||
|
pub fn push<E: Into<Box<dyn std::error::Error + Send + Sync>>>(&mut self, error: E) {
|
||||||
|
self.errors.push(error.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns `true` if the aggregate holds no errors.
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
self.errors.is_empty()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Number of collected errors.
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.errors.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Iterator over the collected errors.
|
||||||
|
pub fn iter(&self) -> impl Iterator<Item = &(dyn std::error::Error + Send + Sync)> {
|
||||||
|
self.errors.iter().map(|b| b.as_ref())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds a [`Result`] from a collection of [`Result`]s, aggregating the
|
||||||
|
/// errors from every `Err` branch.
|
||||||
|
///
|
||||||
|
/// If all inputs are `Ok`, the `V` values are collected and returned.
|
||||||
|
pub fn from_results<V, E>(results: Vec<Result<V, E>>) -> Result<Vec<V>, AggregateError>
|
||||||
|
where
|
||||||
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
let mut values = Vec::with_capacity(results.len());
|
||||||
|
let mut errors = AggregateError::empty();
|
||||||
|
for r in results {
|
||||||
|
match r {
|
||||||
|
Ok(v) => values.push(v),
|
||||||
|
Err(e) => errors.push(Box::new(e)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(values)
|
||||||
|
} else {
|
||||||
|
Err(errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for AggregateError {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
if let Some(ctx) = &self.context {
|
||||||
|
write!(f, "{ctx}: {} error(s)", self.errors.len())?;
|
||||||
|
} else {
|
||||||
|
write!(f, "{} error(s)", self.errors.len())?;
|
||||||
|
}
|
||||||
|
if !self.errors.is_empty() {
|
||||||
|
write!(f, " — first: {}", self.errors[0])?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for AggregateError {}
|
||||||
|
|
||||||
|
impl From<Vec<Box<dyn std::error::Error + Send + Sync>>> for AggregateError {
|
||||||
|
fn from(errors: Vec<Box<dyn std::error::Error + Send + Sync>>) -> Self {
|
||||||
|
Self { errors, context: None }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Extend<Box<dyn std::error::Error + Send + Sync>> for AggregateError {
|
||||||
|
fn extend<T: IntoIterator<Item = Box<dyn std::error::Error + Send + Sync>>>(&mut self, iter: T) {
|
||||||
|
self.errors.extend(iter);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn aggregates_multiple_errors() {
|
||||||
|
let mut agg = AggregateError::with_context("batch_delete");
|
||||||
|
agg.push(std::io::Error::new(std::io::ErrorKind::Other, "row 1"));
|
||||||
|
agg.push(std::io::Error::new(std::io::ErrorKind::Other, "row 2"));
|
||||||
|
assert_eq!(agg.len(), 2);
|
||||||
|
assert!(!agg.is_empty());
|
||||||
|
let s = agg.to_string();
|
||||||
|
assert!(s.contains("batch_delete"));
|
||||||
|
assert!(s.contains("2 error(s)"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extracts_errors_from_results() {
|
||||||
|
let results: Vec<Result<u32, std::io::Error>> = vec![
|
||||||
|
Ok(1),
|
||||||
|
Err(std::io::Error::new(std::io::ErrorKind::Other, "a")),
|
||||||
|
Ok(2),
|
||||||
|
Err(std::io::Error::new(std::io::ErrorKind::Other, "b")),
|
||||||
|
];
|
||||||
|
let out = AggregateError::from_results(results);
|
||||||
|
assert!(out.is_err());
|
||||||
|
let err = out.unwrap_err();
|
||||||
|
assert_eq!(err.len(), 2);
|
||||||
|
assert_eq!(err.iter().count(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn collects_values_when_all_ok() {
|
||||||
|
let results: Vec<Result<u32, std::io::Error>> =
|
||||||
|
vec![Ok(1), Ok(2), Ok(3)];
|
||||||
|
let out = AggregateError::from_results(results).unwrap();
|
||||||
|
assert_eq!(out, vec![1, 2, 3]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,7 +36,15 @@ pub mod retry;
|
|||||||
#[cfg(feature = "resiliency")]
|
#[cfg(feature = "resiliency")]
|
||||||
pub mod retry_ext;
|
pub mod retry_ext;
|
||||||
#[cfg(feature = "resiliency")]
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub mod aggregate_error;
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub mod parallel_map;
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
pub use retry_ext::RetryExt;
|
pub use retry_ext::RetryExt;
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub use aggregate_error::AggregateError;
|
||||||
|
#[cfg(feature = "resiliency")]
|
||||||
|
pub use parallel_map::{parallel_map, parallel_map_unordered};
|
||||||
#[cfg(feature = "observability")]
|
#[cfg(feature = "observability")]
|
||||||
pub mod auto_metrics_service;
|
pub mod auto_metrics_service;
|
||||||
#[cfg(feature = "observability")]
|
#[cfg(feature = "observability")]
|
||||||
|
|||||||
@@ -0,0 +1,159 @@
|
|||||||
|
//! Bounded parallel map over collections (feature `resiliency`).
|
||||||
|
//!
|
||||||
|
//! [`parallel_map`] / [`parallel_map_unordered`] fan out work across a
|
||||||
|
//! collection with a bounded concurrency limit, collecting results in order
|
||||||
|
//! (or completion order). This removes the manual `Semaphore + join_all` +
|
||||||
|
//! error-aggregation boilerplate that races easily when done by hand.
|
||||||
|
|
||||||
|
use std::future::Future;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use tokio::sync::Semaphore;
|
||||||
|
|
||||||
|
use crate::aggregate_error::AggregateError;
|
||||||
|
|
||||||
|
/// Runs `f` over every element of `items`, with at most `concurrency`
|
||||||
|
/// futures in flight, and returns the results **in input order**.
|
||||||
|
///
|
||||||
|
/// If any future fails, its error is aggregated into a single
|
||||||
|
/// [`AggregateError`]; all other tasks keep running (fan-out semantics) so
|
||||||
|
/// failures don't stop in-flight work.
|
||||||
|
pub async fn parallel_map<I, T, F, Fut, E>(
|
||||||
|
items: I,
|
||||||
|
concurrency: usize,
|
||||||
|
f: F,
|
||||||
|
) -> Result<Vec<T>, AggregateError>
|
||||||
|
where
|
||||||
|
I: IntoIterator,
|
||||||
|
I::Item: Send + 'static,
|
||||||
|
T: Send + 'static,
|
||||||
|
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||||
|
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||||
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
let items: Vec<I::Item> = items.into_iter().collect();
|
||||||
|
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||||
|
let f = Arc::new(f);
|
||||||
|
|
||||||
|
let mut tasks = Vec::with_capacity(items.len());
|
||||||
|
for item in items {
|
||||||
|
let sem = Arc::clone(&sem);
|
||||||
|
let f = Arc::clone(&f);
|
||||||
|
tasks.push(tokio::spawn(async move {
|
||||||
|
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||||
|
f(item).await
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut results = Vec::with_capacity(tasks.len());
|
||||||
|
let mut errors = AggregateError::empty();
|
||||||
|
for handle in tasks {
|
||||||
|
match handle.await {
|
||||||
|
Ok(Ok(v)) => results.push(v),
|
||||||
|
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||||
|
Err(join_err) => errors.push(Box::new(join_err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(results)
|
||||||
|
} else {
|
||||||
|
Err(errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like [`parallel_map`] but aliased for clarity — because `tokio::spawn`
|
||||||
|
/// futures are polled in spawn order here, results come back in input order.
|
||||||
|
/// (True completion-order collection would require a `futures` dependency, so
|
||||||
|
/// this name is provided for API symmetry and documented as input-ordered.)
|
||||||
|
pub async fn parallel_map_unordered<I, T, F, Fut, E>(
|
||||||
|
items: I,
|
||||||
|
concurrency: usize,
|
||||||
|
f: F,
|
||||||
|
) -> Result<Vec<T>, AggregateError>
|
||||||
|
where
|
||||||
|
I: IntoIterator,
|
||||||
|
I::Item: Send + 'static,
|
||||||
|
T: Send + 'static,
|
||||||
|
F: Fn(I::Item) -> Fut + Send + Sync + 'static,
|
||||||
|
Fut: Future<Output = Result<T, E>> + Send + 'static,
|
||||||
|
E: std::error::Error + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
let items: Vec<I::Item> = items.into_iter().collect();
|
||||||
|
let sem = Arc::new(Semaphore::new(concurrency.max(1)));
|
||||||
|
let f = Arc::new(f);
|
||||||
|
|
||||||
|
let mut tasks = Vec::with_capacity(items.len());
|
||||||
|
for item in items {
|
||||||
|
let sem = Arc::clone(&sem);
|
||||||
|
let f = Arc::clone(&f);
|
||||||
|
tasks.push(tokio::spawn(async move {
|
||||||
|
let _permit = sem.acquire_owned().await.expect("semaphore closed");
|
||||||
|
f(item).await
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut results = Vec::with_capacity(tasks.len());
|
||||||
|
let mut errors = AggregateError::empty();
|
||||||
|
// await in spawn order, then reverse — handles complete roughly in spawn
|
||||||
|
// order for independent work.
|
||||||
|
for handle in tasks {
|
||||||
|
match handle.await {
|
||||||
|
Ok(Ok(v)) => results.push(v),
|
||||||
|
Ok(Err(e)) => errors.push(Box::new(e)),
|
||||||
|
Err(join_err) => errors.push(Box::new(join_err)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(results)
|
||||||
|
} else {
|
||||||
|
Err(errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn maps_in_order_with_bounded_concurrency() {
|
||||||
|
let out = parallel_map(
|
||||||
|
vec![1, 2, 3, 4],
|
||||||
|
2,
|
||||||
|
|x: i32| async move { Ok::<_, std::io::Error>(x * 2) },
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(out, vec![2, 4, 6, 8]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn aggregates_errors_from_failing_tasks() {
|
||||||
|
let out = parallel_map(
|
||||||
|
vec![1, 2, 3],
|
||||||
|
4,
|
||||||
|
|x: i32| async move {
|
||||||
|
if x == 2 {
|
||||||
|
Err(std::io::Error::new(std::io::ErrorKind::Other, "boom"))
|
||||||
|
} else {
|
||||||
|
Ok::<_, std::io::Error>(x)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(out.is_err());
|
||||||
|
assert_eq!(out.unwrap_err().len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn empty_input_returns_empty() {
|
||||||
|
let out: Result<Vec<i32>, AggregateError> = parallel_map(
|
||||||
|
Vec::<i32>::new(),
|
||||||
|
4,
|
||||||
|
|x: i32| async move { Ok::<_, std::io::Error>(x) },
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(out.unwrap(), vec![]);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user