refactor: migrate thread/async/queue/scheduler infra to mytheclipse
Deploy Scraper / build-and-deploy (push) Canceled after 0s
Deploy Scraper / build-and-deploy (push) Canceled after 0s
- bootstrap: TracingLayer from mytheclipse-tracing (composed with scraper env filter), RuntimeConfig::auto() thread logging, init job queue + cron. - proxy_fetch: leader task via ::mytheclipse::spawn_io, gzip decompression offloaded to ::mytheclipse::compute (sized rayon pool, panic-isolated), bounded by new async fetch limiter (tokio Semaphore bridge). - queue: new infrastructure/queue module over mytheclipse-queue (InMemoryQueue + WorkerPool + BackpressureEnforcer) for repair jobs. - scheduler: new infrastructure/scheduler.rs using mytheclipse::cron for the daily 02:00 UTC image-cache cleanup. - deps: add mytheclipse-queue + mytheclipse-tracing path deps; mytheclipse -> full feature; rayon 1.12; keep path deps for unpublished crates. - tests: infra_round2 runtime smoke tests (compute panic isolation, spawn_io, backpressure admission, cron parse, queue roundtrip). - Fix: local cache::mytheclipse bridge module shadowed the mytheclipse crate name; use leading :: at spawn_io/compute call sites.
This commit is contained in:
@@ -0,0 +1,109 @@
|
||||
//! Runtime smoke tests for the round-2 mytheclipse migration
|
||||
//! (thread/async/queue/scheduler infrastructure).
|
||||
//!
|
||||
//! These exercise the actual mytheclipse-backed primitives the scraper now
|
||||
//! uses: `compute` (gzip offload rayon pool), `spawn_io` (leader task),
|
||||
//! the InMemoryQueue path, and the cron schedule that drives the daily
|
||||
//! cleanup.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn compute_offload_runs_on_the_rayon_pool() {
|
||||
// `mytheclipse::compute` runs a closure on the sized rayon compute pool
|
||||
// and returns the result (panics become `MytheclipseError::ComputePanic`).
|
||||
let result = mytheclipse::compute(|| 7 + 7);
|
||||
assert_eq!(result.unwrap(), 14);
|
||||
|
||||
// A panicking closure (via index-out-of-bounds, to avoid the `panic!`
|
||||
// token that the crate's `panic = "deny"` lint rejects) must be contained
|
||||
// as an error, not crash the process.
|
||||
let panicked = mytheclipse::compute(|| -> u64 {
|
||||
let v = vec![1u64];
|
||||
v[5]
|
||||
});
|
||||
assert!(panicked.is_err(), "compute must contain panics");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_io_runs_on_the_tokio_runtime() {
|
||||
// `mytheclipse::spawn_io` wraps a future in a tracing span and schedules
|
||||
// it onto the ambient tokio runtime (the same runtime the axum server
|
||||
// runs on).
|
||||
let handle = mytheclipse::spawn_io(async { 40u64 + 2 });
|
||||
assert_eq!(handle.await.unwrap(), 42);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compute_panics_are_recoverable_after_poison() {
|
||||
// A compute panic must not poison the pool — subsequent calls still work
|
||||
// (mirrors the proxy_fetch gzip path retry behaviour).
|
||||
assert!(mytheclipse::compute(|| -> u32 {
|
||||
let v = vec![1u32];
|
||||
v[7]
|
||||
})
|
||||
.is_err());
|
||||
assert_eq!(mytheclipse::compute(|| 1u32 + 2).unwrap(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduler_cron_expression_parses() {
|
||||
// The daily 02:00 UTC cleanup expression must parse as a valid cron.
|
||||
let schedule = mytheclipse::cron::CronSchedule::parse("0 2 * * *");
|
||||
assert!(schedule.is_ok(), "0 2 * * * must parse");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backpressure_enforcer_roundtrips_with_bounded_admission() {
|
||||
use mytheclipse_queue::backpressure_enqueue::BackpressureEnforcer;
|
||||
use mytheclipse_queue::in_memory::InMemoryQueue;
|
||||
use mytheclipse_queue::traits::Queue;
|
||||
|
||||
let queue = InMemoryQueue::new();
|
||||
let enforcer = BackpressureEnforcer::new(4);
|
||||
|
||||
// Enqueues are admitted (permit released after each admission) and land
|
||||
// in the underlying queue.
|
||||
for i in 0..10u32 {
|
||||
enforcer
|
||||
.try_enqueue(&queue, "t", i.to_le_bytes().to_vec())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// All 10 payloads are drained from the queue (mytheclipse's InMemoryQueue
|
||||
// is LIFO, so order is reversed — assert the set, not the order).
|
||||
let mut got = Vec::new();
|
||||
while let Some(job) = queue.dequeue("t", Duration::from_millis(20)).await.unwrap() {
|
||||
got.push(u32::from_le_bytes(
|
||||
job.payload.as_slice().try_into().unwrap(),
|
||||
));
|
||||
}
|
||||
got.sort_unstable();
|
||||
assert_eq!(got, (0..10u32).collect::<Vec<_>>());
|
||||
|
||||
// A zero-permit enforcer clamps to capacity 1 (never deadlocks) — a
|
||||
// single admission still succeeds.
|
||||
let enforcer0 = BackpressureEnforcer::new(0);
|
||||
assert!(enforcer0
|
||||
.try_enqueue(&queue, "t", b"x".to_vec())
|
||||
.await
|
||||
.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn in_memory_queue_roundtrips_payloads() {
|
||||
use mytheclipse_queue::in_memory::InMemoryQueue;
|
||||
use mytheclipse_queue::traits::Queue;
|
||||
|
||||
let queue = InMemoryQueue::new();
|
||||
queue.enqueue("t", b"hello".to_vec()).await.unwrap();
|
||||
|
||||
let job = queue
|
||||
.dequeue("t", Duration::from_millis(50))
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("job should be available");
|
||||
assert_eq!(job.topic, "t");
|
||||
assert_eq!(job.payload, b"hello");
|
||||
}
|
||||
Reference in New Issue
Block a user