MythEclipse/zesdex
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: implement retry logic with exponential backoff and jitter for subagent and provider calls

Browse Source
This commit is contained in:
asepharyana
2026-07-18 03:18:27 +07:00
parent b3c5b2a57b
commit b02754acd2
3 changed files with 459 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/zesdex-backend/src/app/subagent/engine.rs
+125 -63
View File
@@ -15,9 +15,49 @@ use crate::dto::chat::message::ChatMessage;
use crate::dto::provider::request::ToolDef;
use crate::tool::tool_is_risky;
use sha2::Digest;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::sync::mpsc;
use zesdex_cms::domain::repository::EditLogRepository;
/// Tiny jitter helper so retry backoffs don't arrive in lockstep.
fn retry_jitter_ns(range_ns: u64) -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.subsec_nanos() as u64
% range_ns
}
/// Exponential backoff with ±25% jitter for subagent step retries.
fn step_retry_delay(attempt: u32) -> Duration {
let base_secs = (2u64).pow(attempt).min(16); // 2s, 4s, 8s, 16s cap
let quarter = (base_secs * 250_000_000).max(100_000_000);
let offset = retry_jitter_ns(quarter);
let ns = base_secs * 1_000_000_000 + offset - quarter / 2;
Duration::from_nanos(ns)
}
/// Heuristic to decide whether the error is worth retrying.
fn should_retry_subagent_step(err_str: &str) -> bool {
let lower = err_str.to_lowercase();
// Never retry auth/billing failures
if err_str.contains("API error 401")
|| err_str.contains("API error 402")
|| err_str.contains("API error 403")
|| lower.contains("unauthorized")
|| lower.contains("forbidden")
|| lower.contains("authentication failed")
{
return false;
}
// Never retry abort or user cancellation
if lower.contains("aborted") {
return false;
}
// Everything else (timeout, 5xx, rate-limit, network blip) is retryable
true
}
fn format_subagent_progress(prefix: &str, text: &str) -> String {
let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();
if lines.is_empty() {
@@ -118,71 +158,93 @@ pub fn run_subagent(
// Use streaming API so the abort flag is checked per SSE event,
// making the subagent responsive to cancellation even during an
// LLM call (non-streaming would block for 10-30s unchecked).
let stream_result = client.chat_with_tools_streaming(
&messages,
tdefs_opt.clone(),
Some(0.7),
Some(4096),
|event| -> bool {
// Check abort on every SSE event for responsive cancellation.
if ctx
.abort_flag
.as_ref()
.is_some_and(|f| f.load(std::sync::atomic::Ordering::SeqCst))
{
return false; // signals provider to abort
}
match event {
crate::app::runtime::stream::StreamEvent::Reasoning(text) => {
current_thinking.push_str(text);
let prog = format_subagent_progress("thinking", &current_thinking);
let _ = tx_clone.blocking_send(SubagentEvent::Progress(prog));
}
crate::app::runtime::stream::StreamEvent::Token(text) => {
current_token.push_str(text);
let prog = format_subagent_progress("replying", &current_token);
let _ = tx_clone.blocking_send(SubagentEvent::Progress(prog));
}
crate::app::runtime::stream::StreamEvent::Usage {
prompt_tokens,
completion_tokens,
..
} => {
// Capture usage so the drain thread can route it
// to the parent's `UsageStats::review_tokens`.
// Last writer wins — providers send exactly one
// Usage event per streaming call.
step_usage = Some((*prompt_tokens, *completion_tokens));
}
_ => {}
}
true
},
);
// Retry the LLM call at the step level (up to 3 attempts) so a
// transient network blip doesn't kill the subagent. The underlying
// `chat_with_tools_streaming` already has its own retry loop (5 +
// non-streaming fallback), so this loop is a second safety net for
// rare cases where the combined 5+10 retries are all exhausted.
let max_step_retries = 3;
let mut step_attempt = 0u32;
let (response, returned_usage) = match stream_result {
Ok(result) => result,
Err(e) => {
let is_abort = ctx
.abort_flag
.as_ref()
.is_some_and(|f| f.load(std::sync::atomic::Ordering::SeqCst))
|| e.to_string().contains("aborted");
let _ = tx.blocking_send(SubagentEvent::StepFailed {
step,
error: if is_abort {
"subagent aborted by user".to_string()
} else {
e.to_string()
},
});
if is_abort {
anyhow::bail!("subagent aborted by parent at step {step}");
let (response, returned_usage) = loop {
step_attempt += 1;
let stream_result = client.chat_with_tools_streaming(
&messages,
tdefs_opt.clone(),
Some(0.7),
Some(4096),
|event| -> bool {
// Check abort on every SSE event for responsive cancellation.
if ctx
.abort_flag
.as_ref()
.is_some_and(|f| f.load(std::sync::atomic::Ordering::SeqCst))
{
return false; // signals provider to abort
}
match event {
crate::app::runtime::stream::StreamEvent::Reasoning(text) => {
current_thinking.push_str(text);
let prog = format_subagent_progress("thinking", &current_thinking);
let _ = tx_clone.blocking_send(SubagentEvent::Progress(prog));
}
crate::app::runtime::stream::StreamEvent::Token(text) => {
current_token.push_str(text);
let prog = format_subagent_progress("replying", &current_token);
let _ = tx_clone.blocking_send(SubagentEvent::Progress(prog));
}
crate::app::runtime::stream::StreamEvent::Usage {
prompt_tokens,
completion_tokens,
..
} => {
// Capture usage so the drain thread can route it
// to the parent's `UsageStats::review_tokens`.
// Last writer wins — providers send exactly one
// Usage event per streaming call.
step_usage = Some((*prompt_tokens, *completion_tokens));
}
_ => {}
}
true
},
);
match stream_result {
Ok(result) => break result,
Err(e) => {
let err_str = e.to_string();
let is_abort = ctx
.abort_flag
.as_ref()
.is_some_and(|f| f.load(std::sync::atomic::Ordering::SeqCst))
|| err_str.contains("aborted");
if is_abort || !should_retry_subagent_step(&err_str) || step_attempt >= max_step_retries {
let _ = tx.blocking_send(SubagentEvent::StepFailed {
step,
error: if is_abort {
"subagent aborted by user".to_string()
} else {
err_str.clone()
},
});
if is_abort {
anyhow::bail!("subagent aborted by parent at step {step}");
}
anyhow::bail!("subagent call failed at step {step} after {step_attempt} attempt(s): {err_str}");
}
let delay = step_retry_delay(step_attempt);
tracing::warn!(
"[subagent] step {step} attempt {step_attempt}/{max_step_retries} failed: {err_str}. \
retrying in {delay:?}...",
);
let _ = tx.blocking_send(SubagentEvent::Progress(format!(
"retrying step {step} ({step_attempt}/{max_step_retries}) after error…",
)));
std::thread::sleep(delay);
}
// No non-streaming fallback — API must support streaming.
// Non-streaming calls block for up to 1 min without checking
// abort_flag, making cancellation unresponsive.
anyhow::bail!("subagent call failed at step {step}: {e}");
}
};