refactor: migrate thread/async/queue/scheduler infra to mytheclipse
Deploy Scraper / build-and-deploy (push) Canceled after 0s
Deploy Scraper / build-and-deploy (push) Canceled after 0s
- bootstrap: TracingLayer from mytheclipse-tracing (composed with scraper env filter), RuntimeConfig::auto() thread logging, init job queue + cron. - proxy_fetch: leader task via ::mytheclipse::spawn_io, gzip decompression offloaded to ::mytheclipse::compute (sized rayon pool, panic-isolated), bounded by new async fetch limiter (tokio Semaphore bridge). - queue: new infrastructure/queue module over mytheclipse-queue (InMemoryQueue + WorkerPool + BackpressureEnforcer) for repair jobs. - scheduler: new infrastructure/scheduler.rs using mytheclipse::cron for the daily 02:00 UTC image-cache cleanup. - deps: add mytheclipse-queue + mytheclipse-tracing path deps; mytheclipse -> full feature; rayon 1.12; keep path deps for unpublished crates. - tests: infra_round2 runtime smoke tests (compute panic isolation, spawn_io, backpressure admission, cron parse, queue roundtrip). - Fix: local cache::mytheclipse bridge module shadowed the mytheclipse crate name; use leading :: at spawn_io/compute call sites.
This commit is contained in:
+9
-4
@@ -13,10 +13,15 @@ default-run = "scraper"
|
||||
|
||||
# Dependensi yang dibutuhkan saat aplikasi berjalan
|
||||
[dependencies]
|
||||
mytheclipse = { version = "1.20", features = ["resiliency", "traffic"] }
|
||||
# Mytheclipse workspace (user's custom library) — path deps so the scraper
|
||||
# builds against the user's actual current source (1.21.x incl. unpublished
|
||||
# cache redis-0.32 fix). Switch to version deps once crates.io catches up.
|
||||
mytheclipse = { path = "/home/code/mytheclipse/crates/mytheclipse", features = ["full"] }
|
||||
mytheclipse-cache = { path = "/home/code/mytheclipse/crates/mytheclipse-cache", features = ["l2-redis", "cache-aside"] }
|
||||
mytheclipse-config = { version = "1.20" }
|
||||
mytheclipse-event = { version = "1.20" }
|
||||
mytheclipse-config = { path = "/home/code/mytheclipse/crates/mytheclipse-config" }
|
||||
mytheclipse-event = { path = "/home/code/mytheclipse/crates/mytheclipse-event" }
|
||||
mytheclipse-queue = { path = "/home/code/mytheclipse/crates/mytheclipse-queue", features = ["in-memory"] }
|
||||
mytheclipse-tracing = { path = "/home/code/mytheclipse/crates/mytheclipse-tracing", features = ["env"] }
|
||||
|
||||
axum = { version = "0.8.8", features = ["ws", "multipart", "macros"] }
|
||||
tokio = { version = "1.49.0", features = ["full"] }
|
||||
@@ -43,7 +48,7 @@ url = "2.5.8"
|
||||
tower-http = { version = "0.6.8", features = ["fs", "cors", "compression-gzip", "compression-br", "compression-zstd"] }
|
||||
dashmap = "6.1"
|
||||
deadpool-redis = { version = "0.22.1", features = ["serde"] }
|
||||
rayon = "1.11"
|
||||
rayon = "1.12"
|
||||
scraper = "0.25.0"
|
||||
flate2 = "1.1"
|
||||
redis = { version = "0.32.7", features = ["tokio-rustls-comp", "safe_iterators"] }
|
||||
|
||||
Reference in New Issue
Block a user