feat(infra): add tools service with document scanner, image & PDF tools

Implement self-hosted document scanner and media processing tools as
an alternative to CamScanner/ilovepdf without third-party uploads.

Backend: Rust Axum gateway + worker pool with NATS JetStream queue
Frontend: Next.js 16 + shadcn/ui + Tailwind v4 + Framer Motion
Pipeline: Canny edge detection -> DLT homography warp -> Sauvola
binarization -> Hough deskew -> Tesseract OCR -> searchable PDF

Phase 1 (MVP) delivers:
- Document scanner with perspective correction and OCR
- Image compress/resize/convert tools
- PDF merge/split/compress tools
- Real-time WebSocket progress updates
- Rate limiting, auto-cleanup, Prometheus metrics
- Full CI/CD pipeline with Docker multi-stage build

Co-Authored-By: Kilo <kilo@kilo.ai>
This commit is contained in:
asepharyana
2026-07-24 13:08:09 +07:00
co-authored by Kilo
parent 9c12d135e4
commit 67288c8723
102 changed files with 11527 additions and 3 deletions
+3816
View File
File diff suppressed because it is too large Load Diff
+37
View File
@@ -0,0 +1,37 @@
[workspace]
resolver = "2"
members = [
"common",
"gateway",
"workers",
"wasm",
]
default-members = [
"common",
"gateway",
"workers",
]
[workspace.package]
version = "0.1.0"
edition = "2021"
license = "MIT"
[workspace.dependencies]
serde = { version = "1", features = ["derive"] }
serde_json = "1"
uuid = { version = "1", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
tokio = { version = "1", features = ["full"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] }
thiserror = "2"
async-nats = "0.39"
redis = { version = "0.28", features = ["tokio-comp", "connection-manager", "aio"] }
image = "0.25"
imageproc = "0.25"
lopdf = "0.36"
reqwest = { version = "0.12", features = ["json"] }
anyhow = "1"
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "tools-common"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
chrono.workspace = true
thiserror.workspace = true
tracing.workspace = true
async-nats.workspace = true
redis.workspace = true
+104
View File
@@ -0,0 +1,104 @@
use thiserror::Error;
/// Errors that can occur during file upload.
#[derive(Debug, Error)]
pub enum UploadError {
#[error("Invalid MIME type: {0}")]
InvalidMime(String),
#[error("File too large: {0} bytes exceeds maximum of {1} bytes")]
FileTooLarge(u64, u64),
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("Virus or suspicious content detected")]
VirusDetected,
#[error("Invalid tool: {0}")]
InvalidTool(String),
#[error("Missing file in upload")]
MissingFile,
#[error("Missing tool parameter")]
MissingTool,
#[error("Serialization error: {0}")]
Serde(#[from] serde_json::Error),
}
/// Errors during processing pipeline execution.
#[derive(Debug, Error)]
pub enum PipelineError {
#[error("Failed to load image: {0}")]
ImageLoad(String),
#[error("Edge detection failed: {0}")]
EdgeDetection(String),
#[error("Corner detection failed: {0}")]
CornerDetection(String),
#[error("Perspective warp failed: {0}")]
Warp(String),
#[error("Shadow removal failed: {0}")]
ShadowRemoval(String),
#[error("Binarization failed: {0}")]
Binarization(String),
#[error("OCR processing failed: {0}")]
Ocr(String),
#[error("PDF generation failed: {0}")]
PdfGeneration(String),
#[error("Pipeline timed out")]
Timeout,
#[error("Internal error: {0}")]
Internal(String),
}
/// Errors related to NATS messaging.
#[derive(Debug, Error)]
pub enum NatsError {
#[error("Failed to publish message: {0}")]
Publish(String),
#[error("Failed to subscribe: {0}")]
Subscribe(String),
#[error("JetStream error: {0}")]
JetStream(String),
#[error("Connection timeout")]
Timeout,
#[error("NATS connection error: {0}")]
Connection(String),
}
/// Errors related to Redis operations.
#[derive(Debug, Error)]
pub enum RedisError {
#[error("Redis connection failed: {0}")]
Connection(String),
#[error("Redis query failed: {0}")]
Query(String),
#[error("Serialization error: {0}")]
Serialization(#[from] serde_json::Error),
#[error("Key not found: {0}")]
NotFound(String),
}
impl From<redis::RedisError> for RedisError {
fn from(e: redis::RedisError) -> Self {
RedisError::Query(e.to_string())
}
}
+3
View File
@@ -0,0 +1,3 @@
pub mod error;
pub mod nats;
pub mod types;
+104
View File
@@ -0,0 +1,104 @@
/// NATS subject constants for the tools service.
///
/// Subject naming convention:
/// tools.<tool_group>.jobs.{job_id} — Job submission queue
/// tools.<tool_group>.progress.{job_id} — Progress update fan-out
/// tools.scheduler.cleanup — Cron-triggered cleanup
// ── Job Subjects ──
pub const SCAN_JOBS: &str = "tools.scan.jobs";
pub const SCAN_PROGRESS: &str = "tools.scan.progress";
pub const IMAGE_JOBS: &str = "tools.image.jobs";
pub const IMAGE_PROGRESS: &str = "tools.image.progress";
pub const PDF_JOBS: &str = "tools.pdf.jobs";
pub const PDF_PROGRESS: &str = "tools.pdf.progress";
pub const VIDEO_JOBS: &str = "tools.video.jobs";
pub const VIDEO_PROGRESS: &str = "tools.video.progress";
pub const AUDIO_JOBS: &str = "tools.audio.jobs";
pub const AUDIO_PROGRESS: &str = "tools.audio.progress";
// ── Scheduler Subjects ──
pub const SCHEDULER_CLEANUP: &str = "tools.scheduler.cleanup";
// ── Stream Names ──
pub const STREAM_JOBS: &str = "tools-jobs";
pub const STREAM_PROGRESS: &str = "tools-progress";
// ── Stream Configuration ──
/// Returns the stream configuration for jobs.
/// Max age: 24h, storage: file (persistent on disk).
pub fn jobs_stream_config() -> async_nats::jetstream::stream::Config {
use async_nats::jetstream::stream::Config;
Config {
name: STREAM_JOBS.to_string(),
subjects: vec![
"tools.scan.jobs.*".to_string(),
"tools.image.jobs.*".to_string(),
"tools.pdf.jobs.*".to_string(),
"tools.video.jobs.*".to_string(),
"tools.audio.jobs.*".to_string(),
"tools.scheduler.>".to_string(),
],
max_age: std::time::Duration::from_secs(24 * 3600),
storage: async_nats::jetstream::stream::StorageType::File,
..Default::default()
}
}
/// Returns the stream configuration for progress events.
/// Max age: 1h, storage: memory (no persistence needed).
pub fn progress_stream_config() -> async_nats::jetstream::stream::Config {
use async_nats::jetstream::stream::Config;
Config {
name: STREAM_PROGRESS.to_string(),
subjects: vec![
"tools.scan.progress.*".to_string(),
"tools.image.progress.*".to_string(),
"tools.pdf.progress.*".to_string(),
"tools.video.progress.*".to_string(),
"tools.audio.progress.*".to_string(),
],
max_age: std::time::Duration::from_secs(3600),
storage: async_nats::jetstream::stream::StorageType::Memory,
..Default::default()
}
}
/// Build a job subject for a given tool and job ID.
pub fn job_subject(tool_group: &str, job_id: &str) -> String {
format!("tools.{}.jobs.{}", tool_group, job_id)
}
/// Build a progress subject for a given tool and job ID.
pub fn progress_subject(tool_group: &str, job_id: &str) -> String {
format!("tools.{}.progress.{}", tool_group, job_id)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_subject_format() {
assert_eq!(job_subject("scan", "abc-123"), "tools.scan.jobs.abc-123");
assert_eq!(
progress_subject("scan", "abc-123"),
"tools.scan.progress.abc-123"
);
assert_eq!(
job_subject("image", "def-456"),
"tools.image.jobs.def-456"
);
assert_eq!(SCHEDULER_CLEANUP, "tools.scheduler.cleanup");
}
#[test]
fn test_stream_names() {
assert_eq!(STREAM_JOBS, "tools-jobs");
assert_eq!(STREAM_PROGRESS, "tools-progress");
}
}
+228
View File
@@ -0,0 +1,228 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
/// Status of a processing job.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum JobStatus {
Queued,
Processing {
stage: String,
progress: u8,
},
Completed,
NeedsManualCrop,
Failed(String),
}
/// Available tool types.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub enum Tool {
Scan,
ImageCompress,
ImageResize,
ImageConvert,
RemoveBg,
PdfMerge,
PdfSplit,
ImagesToPdf,
PdfCompress,
PdfToImages,
VideoCompress,
AudioExtract,
VideoTrim,
GifMaker,
AudioConvert,
}
impl Tool {
/// Returns the NATS subject prefix for this tool.
pub fn subject_prefix(&self) -> &'static str {
match self {
Tool::Scan => "tools.scan",
Tool::ImageCompress
| Tool::ImageResize
| Tool::ImageConvert
| Tool::RemoveBg => "tools.image",
Tool::PdfMerge
| Tool::PdfSplit
| Tool::ImagesToPdf
| Tool::PdfCompress
| Tool::PdfToImages => "tools.pdf",
Tool::VideoCompress | Tool::VideoTrim | Tool::GifMaker => "tools.video",
Tool::AudioExtract | Tool::AudioConvert => "tools.audio",
}
}
pub fn as_str(&self) -> &'static str {
match self {
Tool::Scan => "scan",
Tool::ImageCompress => "image-compress",
Tool::ImageResize => "image-resize",
Tool::ImageConvert => "image-convert",
Tool::RemoveBg => "remove-bg",
Tool::PdfMerge => "pdf-merge",
Tool::PdfSplit => "pdf-split",
Tool::ImagesToPdf => "images-to-pdf",
Tool::PdfCompress => "pdf-compress",
Tool::PdfToImages => "pdf-to-images",
Tool::VideoCompress => "video-compress",
Tool::AudioExtract => "audio-extract",
Tool::VideoTrim => "video-trim",
Tool::GifMaker => "gif-maker",
Tool::AudioConvert => "audio-convert",
}
}
pub fn from_str(s: &str) -> Option<Self> {
match s {
"scan" => Some(Tool::Scan),
"image-compress" => Some(Tool::ImageCompress),
"image-resize" => Some(Tool::ImageResize),
"image-convert" => Some(Tool::ImageConvert),
"remove-bg" => Some(Tool::RemoveBg),
"pdf-merge" => Some(Tool::PdfMerge),
"pdf-split" => Some(Tool::PdfSplit),
"images-to-pdf" => Some(Tool::ImagesToPdf),
"pdf-compress" => Some(Tool::PdfCompress),
"pdf-to-images" => Some(Tool::PdfToImages),
"video-compress" => Some(Tool::VideoCompress),
"audio-extract" => Some(Tool::AudioExtract),
"video-trim" => Some(Tool::VideoTrim),
"gif-maker" => Some(Tool::GifMaker),
"audio-convert" => Some(Tool::AudioConvert),
_ => None,
}
}
}
/// Options for document scanning.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScanOptions {
pub ocr: bool,
pub enhance: bool,
pub output_format: OutputFormat,
pub dpi: u32,
pub quality: u8,
pub language: String,
pub color_mode: ColorMode,
pub page_size: PageSize,
}
impl Default for ScanOptions {
fn default() -> Self {
Self {
ocr: true,
enhance: true,
output_format: OutputFormat::Pdf,
dpi: 300,
quality: 90,
language: "eng+ind".to_string(),
color_mode: ColorMode::BlackAndWhite,
page_size: PageSize::A4,
}
}
}
/// Options for image tools.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ImageOptions {
pub quality: Option<u8>,
pub width: Option<u32>,
pub height: Option<u32>,
pub format: Option<String>,
pub fit: Option<String>,
pub bg_color: Option<[u8; 3]>,
}
/// Options for PDF tools.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PdfOptions {
pub quality: Option<u8>,
pub pages: Option<String>,
pub dpi: Option<u32>,
pub page_size: Option<PageSize>,
pub margin_mm: Option<u32>,
}
/// A complete job record.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Job {
pub id: Uuid,
pub tool: Tool,
pub status: JobStatus,
pub file_path: String,
pub result_path: Option<String>,
pub file_size: u64,
pub options: serde_json::Value,
pub created_at: DateTime<Utc>,
pub ttl_seconds: u64,
}
/// Progress update sent via NATS and forwarded via WebSocket.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JobProgress {
pub job_id: Uuid,
pub status: JobStatus,
pub stage: String,
pub progress: u8,
pub message: String,
}
/// Response returned after successful upload.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UploadResponse {
pub job_id: Uuid,
pub status: String,
pub tool: String,
pub ws_url: String,
pub created_at: DateTime<Utc>,
pub estimated_seconds: u8,
}
/// Job status response.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JobStatusResponse {
pub job_id: Uuid,
pub status: String,
pub tool: String,
pub progress: u8,
pub stage: String,
pub message: String,
pub result: Option<ResultInfo>,
pub created_at: DateTime<Utc>,
pub error: Option<String>,
}
/// Result metadata included in status response.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResultInfo {
pub download_url: String,
pub file_size: u64,
pub file_name: String,
pub preview_url: Option<String>,
}
/// Output format for scan results.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum OutputFormat {
Pdf,
Jpeg,
Png,
}
/// Color mode for processed output.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum ColorMode {
BlackAndWhite,
Grayscale,
Color,
}
/// Page size for PDF output.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum PageSize {
A4,
Letter,
Auto,
}
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "tools-gateway"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
tools-common = { path = "../common" }
axum = { version = "0.8", features = ["multipart", "ws"] }
tower = "0.5"
tower-http = { version = "0.6", features = ["cors", "trace", "limit"] }
tokio.workspace = true
tokio-util = { version = "0.7", features = ["io"] }
serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
chrono.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
async-nats.workspace = true
redis.workspace = true
thiserror.workspace = true
anyhow.workspace = true
futures = "0.3"
+76
View File
@@ -0,0 +1,76 @@
use std::path::PathBuf;
/// Application configuration loaded from environment variables.
#[derive(Debug, Clone)]
pub struct AppConfig {
pub port: u16,
pub nats_url: String,
pub redis_url: String,
pub storage_path: PathBuf,
pub max_file_size_mb: u64,
pub job_ttl_seconds: u64,
pub rate_limit_per_minute: u32,
pub rust_log: String,
}
impl AppConfig {
/// Load configuration from environment variables with sensible defaults.
pub fn from_env() -> Self {
Self {
port: env_or_default("GATEWAY_PORT", "3001")
.parse()
.unwrap_or(3001),
nats_url: env_or_default("NATS_URL", "nats://localhost:4222"),
redis_url: env_or_default("REDIS_URL", "redis://localhost:6379"),
storage_path: PathBuf::from(env_or_default("STORAGE_PATH", "/data/tools")),
max_file_size_mb: env_or_default("MAX_FILE_SIZE_MB", "50")
.parse()
.unwrap_or(50),
job_ttl_seconds: env_or_default("JOB_TTL_SECONDS", "3600")
.parse()
.unwrap_or(3600),
rate_limit_per_minute: env_or_default("RATE_LIMIT_PER_MINUTE", "30")
.parse()
.unwrap_or(30),
rust_log: env_or_default("RUST_LOG", "info"),
}
}
pub fn max_file_size_bytes(&self) -> u64 {
self.max_file_size_mb * 1024 * 1024
}
}
fn env_or_default(key: &str, default: &str) -> String {
std::env::var(key).unwrap_or_else(|_| default.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_default_config() {
let config = AppConfig::from_env();
assert_eq!(config.port, 3001);
assert_eq!(config.nats_url, "nats://localhost:4222");
assert_eq!(config.redis_url, "redis://localhost:6379");
assert_eq!(config.max_file_size_mb, 50);
assert_eq!(config.job_ttl_seconds, 3600);
assert_eq!(config.rate_limit_per_minute, 30);
}
#[test]
fn test_file_size_bytes() {
let config = AppConfig::from_env();
assert_eq!(config.max_file_size_bytes(), 50 * 1024 * 1024);
}
#[test]
fn test_env_override() {
std::env::set_var("GATEWAY_PORT", "9999");
let config = AppConfig::from_env();
assert_eq!(config.port, 9999);
std::env::remove_var("GATEWAY_PORT");
}
}
+143
View File
@@ -0,0 +1,143 @@
use std::net::SocketAddr;
use std::sync::Arc;
use axum::{
routing::{get, post},
Router,
};
use tower_http::cors::{Any, CorsLayer};
use tower_http::limit::RequestBodyLimitLayer;
use tower_http::trace::TraceLayer;
use tracing_subscriber::EnvFilter;
mod config;
mod metrics;
mod middleware;
mod nats;
mod redis;
mod routes;
use config::AppConfig;
use metrics::Metrics;
use routes::health::AppState;
#[tokio::main]
async fn main() {
// Load config
let config = AppConfig::from_env();
// Init logging
tracing_subscriber::fmt()
.with_env_filter(EnvFilter::new(&config.rust_log))
.init();
tracing::info!("Starting tools-gateway...");
// Init Redis client
let redis_client = redis::create_client(&config.redis_url)
.expect("Failed to create Redis client");
tracing::info!("Redis client created for {}", config.redis_url);
// Init NATS connection
let nats = nats::publisher::NatsPublisher::connect(&config.nats_url)
.await
.expect("Failed to connect to NATS");
tracing::info!("Connected to NATS at {}", config.nats_url);
// Ensure NATS streams exist
if let Err(e) = ensure_nats_streams(&nats).await {
tracing::warn!("Failed to create NATS streams: {}", e);
}
// Init metrics
let metrics = Metrics::new();
// Shared state
let state = Arc::new(AppState {
redis: redis_client,
nats,
config: config.clone(),
metrics,
});
// Build router
let app = Router::new()
.route("/api/upload", post(routes::upload::upload_handler))
.route("/api/job/{id}", get(routes::job::job_status_handler))
.route(
"/api/job/{id}/preview",
get(routes::job::job_preview_handler),
)
.route("/api/job/{id}/ws", get(routes::ws::ws_handler))
.route("/api/download/{id}", get(routes::download::download_handler))
.route("/health", get(routes::health::health_handler))
.route("/metrics", get(routes::health::metrics_handler))
.layer(TraceLayer::new_for_http())
.layer(CorsLayer::new().allow_origin(Any))
.layer(RequestBodyLimitLayer::new(
((config.max_file_size_mb + 1) * 1024 * 1024) as usize,
))
.with_state(state);
// Start server
let addr = SocketAddr::from(([0, 0, 0, 0], config.port));
tracing::info!("Gateway listening on {}", addr);
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal())
.await
.unwrap();
}
/// Ensure required NATS JetStream streams exist.
async fn ensure_nats_streams(
nats: &async_nats::Client,
) -> Result<(), Box<dyn std::error::Error>> {
let js = async_nats::jetstream::new(nats.clone());
match js
.get_or_create_stream(tools_common::nats::jobs_stream_config())
.await
{
Ok(_) => tracing::info!("NATS stream 'tools-jobs' ready"),
Err(e) => tracing::warn!("Failed to create tools-jobs stream: {}", e),
}
match js
.get_or_create_stream(tools_common::nats::progress_stream_config())
.await
{
Ok(_) => tracing::info!("NATS stream 'tools-progress' ready"),
Err(e) => tracing::warn!("Failed to create tools-progress stream: {}", e),
}
Ok(())
}
/// Handle graceful shutdown on SIGINT/SIGTERM.
async fn shutdown_signal() {
let ctrl_c = async {
tokio::signal::ctrl_c()
.await
.expect("Failed to install Ctrl+C handler");
};
#[cfg(unix)]
let terminate = async {
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.expect("Failed to install SIGTERM handler")
.recv()
.await;
};
#[cfg(not(unix))]
let terminate = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {},
_ = terminate => {},
}
tracing::info!("Shutting down gateway...");
}
+181
View File
@@ -0,0 +1,181 @@
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Mutex;
/// Simple Prometheus metrics collector.
pub struct Metrics {
/// Counter: tools_jobs_total{tool, status}
jobs_total: Mutex<HashMap<(String, String), AtomicU64>>,
/// Counter: tools_uploaded_files_total{tool, status}
uploaded_files_total: Mutex<HashMap<(String, String), AtomicU64>>,
/// Histogram buckets for processing duration (ms)
duration_buckets: Vec<f64>,
/// Histogram: tools_processing_duration_ms{tool}
duration_histogram: Mutex<HashMap<String, Vec<AtomicU64>>>,
/// Gauge: tools_queue_depth{tool}
queue_depth: Mutex<HashMap<String, AtomicU64>>,
/// Counter: tools_rate_limit_hits{tool}
rate_limit_hits: Mutex<HashMap<String, AtomicU64>>,
/// Counter: cleanup deleted files
cleanup_deleted_files: AtomicU64,
}
impl Metrics {
pub fn new() -> Self {
Self {
jobs_total: Mutex::new(HashMap::new()),
uploaded_files_total: Mutex::new(HashMap::new()),
duration_buckets: vec![
100.0, 250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 16000.0, 32000.0,
],
duration_histogram: Mutex::new(HashMap::new()),
queue_depth: Mutex::new(HashMap::new()),
rate_limit_hits: Mutex::new(HashMap::new()),
cleanup_deleted_files: AtomicU64::new(0),
}
}
pub fn increment_jobs_total(&self, tool: &str, status: &str) {
if let Ok(mut map) = self.jobs_total.lock() {
map.entry((tool.to_string(), status.to_string()))
.or_insert_with(|| AtomicU64::new(0))
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_uploaded_files(&self, tool: &str, status: &str) {
if let Ok(mut map) = self.uploaded_files_total.lock() {
map.entry((tool.to_string(), status.to_string()))
.or_insert_with(|| AtomicU64::new(0))
.fetch_add(1, Ordering::Relaxed);
}
}
#[allow(unused)]
pub fn record_duration(&self, tool: &str, duration_ms: f64) {
if let Ok(mut map) = self.duration_histogram.lock() {
let entry = map
.entry(tool.to_string())
.or_insert_with(|| {
(0..self.duration_buckets.len())
.map(|_| AtomicU64::new(0))
.collect()
});
for (i, bucket) in self.duration_buckets.iter().enumerate() {
if duration_ms <= *bucket {
if let Some(b) = entry.get(i) {
b.fetch_add(1, Ordering::Relaxed);
}
}
}
}
}
pub fn set_queue_depth(&self, tool: &str, depth: u64) {
if let Ok(mut map) = self.queue_depth.lock() {
map.entry(tool.to_string())
.or_insert_with(|| AtomicU64::new(0))
.store(depth, Ordering::Relaxed);
}
}
#[allow(unused)]
pub fn increment_rate_limit_hits(&self, tool: &str) {
if let Ok(mut map) = self.rate_limit_hits.lock() {
map.entry(tool.to_string())
.or_insert_with(|| AtomicU64::new(0))
.fetch_add(1, Ordering::Relaxed);
}
}
#[allow(unused)]
pub fn increment_cleanup_deleted(&self) {
self.cleanup_deleted_files.fetch_add(1, Ordering::Relaxed);
}
/// Format all metrics as Prometheus text format.
pub fn format(&self) -> String {
let mut output = String::new();
output.push_str("# HELP tools_jobs_total Total jobs processed\n");
output.push_str("# TYPE tools_jobs_total counter\n");
if let Ok(map) = self.jobs_total.lock() {
for ((tool, status), count) in map.iter() {
let val = count.load(Ordering::Relaxed);
output.push_str(&format!(
"tools_jobs_total{{tool=\"{}\",status=\"{}\"}} {}\n",
tool, status, val
));
}
}
output.push_str("# HELP tools_uploaded_files_total Total uploaded files\n");
output.push_str("# TYPE tools_uploaded_files_total counter\n");
if let Ok(map) = self.uploaded_files_total.lock() {
for ((tool, status), count) in map.iter() {
let val = count.load(Ordering::Relaxed);
output.push_str(&format!(
"tools_uploaded_files_total{{tool=\"{}\",status=\"{}\"}} {}\n",
tool, status, val
));
}
}
output.push_str("# HELP tools_processing_duration_ms Processing duration histogram\n");
output.push_str("# TYPE tools_processing_duration_ms histogram\n");
if let Ok(map) = self.duration_histogram.lock() {
for (tool, buckets) in map.iter() {
for (i, bucket) in self.duration_buckets.iter().enumerate() {
if let Some(b) = buckets.get(i) {
let val = b.load(Ordering::Relaxed);
if val > 0 {
output.push_str(&format!(
"tools_processing_duration_ms_bucket{{tool=\"{}\",le=\"{}\"}} {}\n",
tool, bucket, val
));
}
}
}
}
}
output.push_str("# HELP tools_queue_depth Current queue depth\n");
output.push_str("# TYPE tools_queue_depth gauge\n");
if let Ok(map) = self.queue_depth.lock() {
for (tool, depth) in map.iter() {
let val = depth.load(Ordering::Relaxed);
output.push_str(&format!(
"tools_queue_depth{{tool=\"{}\"}} {}\n",
tool, val
));
}
}
output.push_str("# HELP tools_rate_limit_hits Total rate limit violations\n");
output.push_str("# TYPE tools_rate_limit_hits counter\n");
if let Ok(map) = self.rate_limit_hits.lock() {
for (tool, count) in map.iter() {
let val = count.load(Ordering::Relaxed);
output.push_str(&format!(
"tools_rate_limit_hits{{tool=\"{}\"}} {}\n",
tool, val
));
}
}
output.push_str("# HELP tools_cleanup_deleted_files Total files deleted by cleanup\n");
output.push_str("# TYPE tools_cleanup_deleted_files counter\n");
output.push_str(&format!(
"tools_cleanup_deleted_files {}\n",
self.cleanup_deleted_files.load(Ordering::Relaxed)
));
output
}
}
impl Default for Metrics {
fn default() -> Self {
Self::new()
}
}
@@ -0,0 +1,66 @@
use axum::{
http::StatusCode,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;
/// Unified JSON error response format.
#[derive(Debug)]
pub struct AppError {
pub status_code: StatusCode,
pub code: String,
pub message: String,
}
impl AppError {
pub fn bad_request(message: impl Into<String>) -> Self {
Self {
status_code: StatusCode::BAD_REQUEST,
code: "bad_request".to_string(),
message: message.into(),
}
}
pub fn not_found(message: impl Into<String>) -> Self {
Self {
status_code: StatusCode::NOT_FOUND,
code: "not_found".to_string(),
message: message.into(),
}
}
pub fn too_large(message: impl Into<String>) -> Self {
Self {
status_code: StatusCode::PAYLOAD_TOO_LARGE,
code: "file_too_large".to_string(),
message: message.into(),
}
}
pub fn rate_limited(retry_after: u64) -> Self {
Self {
status_code: StatusCode::TOO_MANY_REQUESTS,
code: "rate_limit_exceeded".to_string(),
message: format!("Rate limit exceeded. Retry after {} seconds", retry_after),
}
}
pub fn internal(message: impl Into<String>) -> Self {
Self {
status_code: StatusCode::INTERNAL_SERVER_ERROR,
code: "internal_error".to_string(),
message: message.into(),
}
}
}
impl IntoResponse for AppError {
fn into_response(self) -> Response {
let body = json!({
"error": self.message,
"code": self.code,
});
(self.status_code, Json(body)).into_response()
}
}
@@ -0,0 +1,2 @@
pub mod error_handler;
pub mod request_id;
@@ -0,0 +1,73 @@
use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::task::{Context, Poll};
use axum::{extract::Request, response::Response};
use tower::{Layer, Service};
use uuid::Uuid;
/// Middleware that adds a unique X-Request-Id header to every request.
#[derive(Clone, Default)]
pub struct RequestIdLayer;
impl<S> Layer<S> for RequestIdLayer {
type Service = RequestIdMiddleware<S>;
fn layer(&self, inner: S) -> Self::Service {
RequestIdMiddleware {
inner,
counter: AtomicU64::new(0),
}
}
}
pub struct RequestIdMiddleware<S> {
inner: S,
counter: AtomicU64,
}
impl<S: Clone> Clone for RequestIdMiddleware<S> {
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
counter: AtomicU64::new(self.counter.load(Ordering::Relaxed)),
}
}
}
impl<S, ReqBody, ResBody> Service<Request<ReqBody>> for RequestIdMiddleware<S>
where
S: Service<Request<ReqBody>, Response = Response<ResBody>>,
S::Future: Send + 'static,
S::Error: 'static,
ReqBody: Send + 'static,
ResBody: Default + Send + 'static,
{
type Response = Response<ResBody>;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, req: Request<ReqBody>) -> Self::Future {
let request_id = Uuid::new_v4().to_string();
let (mut parts, body) = req.into_parts();
parts
.headers
.insert("x-request-id", request_id.parse().unwrap());
let req = Request::from_parts(parts, body);
let fut = self.inner.call(req);
Box::pin(async move {
let mut response: Response<ResBody> = fut.await?;
response
.headers_mut()
.insert("x-request-id", request_id.parse().unwrap());
Ok(response)
})
}
}
@@ -0,0 +1 @@
pub mod publisher;
@@ -0,0 +1,64 @@
use async_nats::Client;
use tools_common::error::NatsError;
use tools_common::nats;
use tools_common::types::{Job, JobProgress, Tool};
/// NATS publisher for job and progress messages.
pub struct NatsPublisher;
impl NatsPublisher {
/// Connect to NATS server.
pub async fn connect(url: &str) -> Result<Client, NatsError> {
async_nats::connect(url)
.await
.map_err(|e| NatsError::Connection(e.to_string()))
}
/// Publish a job to the appropriate NATS subject.
pub async fn publish_job(nats: &Client, tool: &Tool, job: &Job) -> Result<(), NatsError> {
let prefix = tool.subject_prefix();
let subject = nats::job_subject(prefix, &job.id.to_string());
let payload = serde_json::to_vec(job)
.map_err(|e| NatsError::Publish(e.to_string()))?;
nats.publish(subject, payload.into())
.await
.map_err(|e| NatsError::Publish(e.to_string()))?;
tracing::debug!(
job_id = %job.id,
tool = %tool.as_str(),
"Published job to NATS"
);
Ok(())
}
/// Publish a progress update to the NATS progress subject.
pub async fn publish_progress(
nats: &Client,
progress: &JobProgress,
) -> Result<(), NatsError> {
let tool_prefix = ""; // We need the tool from somewhere — stored in progress
let subject = format!("tools.*.progress.{}", progress.job_id);
let payload = serde_json::to_vec(progress)
.map_err(|e| NatsError::Publish(e.to_string()))?;
nats.publish(subject, payload.into())
.await
.map_err(|e| NatsError::Publish(e.to_string()))?;
Ok(())
}
/// Subscribe to NATS progress updates for a specific job.
pub async fn subscribe_progress(
nats: &Client,
job_id: &str,
) -> Result<async_nats::Subscriber, NatsError> {
let subject = format!("tools.*.progress.{}", job_id);
nats.subscribe(subject)
.await
.map_err(|e| NatsError::Subscribe(e.to_string()))
}
}
@@ -0,0 +1,78 @@
use redis::{AsyncCommands, RedisError};
use uuid::Uuid;
use tools_common::types::Job;
/// Repository for job CRUD operations on Redis.
pub struct JobRepository;
impl JobRepository {
/// Create a new job record in Redis with TTL.
pub async fn create(
conn: &mut impl AsyncCommands,
job: &Job,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let key = format!("job:{}", job.id);
let json = serde_json::to_string(job)?;
let _: () = conn
.set_ex(key, json, job.ttl_seconds)
.await
.map_err(|e: RedisError| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?;
Ok(())
}
/// Get a job by ID from Redis.
pub async fn get(
conn: &mut impl AsyncCommands,
job_id: Uuid,
) -> Result<Job, Box<dyn std::error::Error + Send + Sync>> {
let key = format!("job:{}", job_id);
let json: String = conn.get(&key).await.map_err(|_| {
Box::new(std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("Job {} not found", job_id),
)) as Box<dyn std::error::Error + Send + Sync>
})?;
let job: Job = serde_json::from_str(&json)?;
Ok(job)
}
/// Update the status of a job in Redis and refresh TTL.
pub async fn update_status(
conn: &mut impl AsyncCommands,
job_id: Uuid,
status: &tools_common::types::JobStatus,
result_path: Option<String>,
ttl_seconds: u64,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let key = format!("job:{}", job_id);
let json: String = conn
.get(&key)
.await
.map_err(|e: RedisError| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?;
let mut job: Job = serde_json::from_str(&json)?;
job.status = status.clone();
if let Some(path) = result_path {
job.result_path = Some(path);
}
let json = serde_json::to_string(&job)?;
let _: () = conn
.set_ex(key, json, ttl_seconds)
.await
.map_err(|e: RedisError| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?;
Ok(())
}
/// Delete a job from Redis.
pub async fn delete(
conn: &mut impl AsyncCommands,
job_id: Uuid,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let key = format!("job:{}", job_id);
let _: usize = conn
.del(key)
.await
.map_err(|e: RedisError| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?;
Ok(())
}
}
@@ -0,0 +1,9 @@
pub mod job;
pub mod ratelimit;
use redis::Client;
/// Create a Redis client.
pub fn create_client(url: &str) -> Result<Client, redis::RedisError> {
Client::open(url)
}
@@ -0,0 +1,50 @@
use redis::AsyncCommands;
/// Sliding window rate limiter using Redis sorted sets.
pub struct RateLimiter;
impl RateLimiter {
/// Check if a request is within the rate limit.
pub async fn check(
conn: &mut impl AsyncCommands,
ip: &str,
tool: &str,
max_per_minute: u32,
) -> Result<bool, Box<dyn std::error::Error>> {
let key = format!("ratelimit:{}:{}", ip, tool);
let now = chrono::Utc::now().timestamp_millis();
let window_start = now - 60_000;
// Remove entries outside the window
let _: usize = conn.zrembyscore(&key, 0, window_start).await?;
// Add current entry
let _: usize = conn
.zadd(&key, format!("{}:{}", ip, now), now as f64)
.await?;
// Set TTL on the key (cleanup)
let _: usize = conn.expire(&key, 120).await?;
// Count entries in window
let count: u32 = conn.zcount(&key, window_start, now).await?;
Ok(count <= max_per_minute)
}
/// Get remaining requests within the current window.
pub async fn remaining(
conn: &mut impl AsyncCommands,
ip: &str,
tool: &str,
max_per_minute: u32,
) -> Result<u32, Box<dyn std::error::Error>> {
let key = format!("ratelimit:{}:{}", ip, tool);
let now = chrono::Utc::now().timestamp_millis();
let window_start = now - 60_000;
let count: u32 = conn.zcount(&key, window_start, now).await?;
Ok(max_per_minute.saturating_sub(count))
}
}
@@ -0,0 +1,121 @@
use std::sync::Arc;
use axum::{
extract::{Path, State},
http::{header, StatusCode},
response::{IntoResponse, Response},
};
use tokio_util::io::ReaderStream;
use uuid::Uuid;
use crate::routes::health::AppState;
use tools_common::types::JobStatus;
/// Handle GET /api/download/{id}
pub async fn download_handler(
State(state): State<Arc<AppState>>,
Path(id): Path<Uuid>,
) -> Result<Response, (StatusCode, JsonResponse)> {
let mut conn = state.redis.get_multiplexed_async_connection().await.map_err(|_| {
(
StatusCode::INTERNAL_SERVER_ERROR,
JsonResponse(serde_json::json!({ "error": "Redis connection failed" })),
)
})?;
let job = crate::redis::job::JobRepository::get(&mut conn, id)
.await
.map_err(|_| {
(
StatusCode::NOT_FOUND,
JsonResponse(serde_json::json!({ "error": "Job not found or expired" })),
)
})?;
// Verify job is completed
if job.status != JobStatus::Completed {
return Err((
StatusCode::BAD_REQUEST,
JsonResponse(serde_json::json!({
"error": "Job is not completed yet",
"status": match job.status {
JobStatus::Queued => "queued",
JobStatus::Processing { .. } => "processing",
JobStatus::NeedsManualCrop => "needs_manual_crop",
JobStatus::Failed(_) => "failed",
_ => "unknown",
}
})),
));
}
let result_path = job.result_path.ok_or_else(|| {
(
StatusCode::NOT_FOUND,
JsonResponse(serde_json::json!({ "error": "Result file path not found" })),
)
})?;
// Open file
let file = tokio::fs::File::open(&result_path).await.map_err(|e| {
(
StatusCode::NOT_FOUND,
JsonResponse(serde_json::json!({ "error": format!("File not found: {}", e) })),
)
})?;
let metadata = file.metadata().await.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
JsonResponse(serde_json::json!({
"error": format!("Failed to read metadata: {}", e)
})),
)
})?;
// Determine content type
let ext = result_path.rsplit('.').next().unwrap_or("bin").to_string();
let content_type = match ext.as_str() {
"pdf" => "application/pdf",
"jpg" | "jpeg" => "image/jpeg",
"png" => "image/png",
"webp" => "image/webp",
"mp4" => "video/mp4",
"mp3" => "audio/mpeg",
"zip" => "application/zip",
_ => "application/octet-stream",
};
// Generate filename for download
let file_name = format!(
"{}_{}.{}",
job.tool.as_str(),
job.id.to_string().split('-').next().unwrap_or("result"),
ext
);
// Stream the file
let stream = ReaderStream::new(file);
let body = axum::body::Body::from_stream(stream);
let response = Response::builder()
.header(header::CONTENT_TYPE, content_type)
.header(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{}\"", file_name),
)
.header(header::CONTENT_LENGTH, metadata.len().to_string())
.body(body)
.unwrap();
Ok(response)
}
/// Wrapper for JSON error responses.
pub struct JsonResponse(pub serde_json::Value);
impl IntoResponse for JsonResponse {
fn into_response(self) -> Response {
(StatusCode::OK, axum::Json(self.0)).into_response()
}
}
@@ -0,0 +1,69 @@
use std::sync::Arc;
use axum::{extract::State, Json};
use redis::AsyncCommands;
use serde::Serialize;
use crate::metrics::Metrics;
/// Shared application state accessible from all handlers.
pub struct AppState {
pub redis: redis::Client,
pub nats: async_nats::Client,
pub config: crate::config::AppConfig,
pub metrics: Metrics,
}
/// Health check response.
#[derive(Serialize)]
pub struct HealthResponse {
pub status: String,
pub version: String,
pub redis: String,
pub nats: String,
pub uptime_seconds: u64,
}
/// Handle GET /health
pub async fn health_handler(
State(state): State<Arc<AppState>>,
) -> Json<HealthResponse> {
let redis_status = {
match state.redis.get_multiplexed_async_connection().await {
Ok(mut conn) => match redis::cmd("PING").query_async::<String>(&mut conn).await {
Ok(_) => "connected".to_string(),
Err(_) => "error".to_string(),
},
Err(_) => "disconnected".to_string(),
}
};
let nats_status = if state
.nats
.publish("tools.health.check", b"ping".to_vec().into())
.await
.is_ok()
{
"connected".to_string()
} else {
"disconnected".to_string()
};
Json(HealthResponse {
status: "ok".to_string(),
version: env!("CARGO_PKG_VERSION").to_string(),
redis: redis_status,
nats: nats_status,
uptime_seconds: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0),
})
}
/// Handle GET /metrics
pub async fn metrics_handler(
State(state): State<Arc<AppState>>,
) -> Result<String, (axum::http::StatusCode, axum::Json<serde_json::Value>)> {
Ok(state.metrics.format())
}
@@ -0,0 +1,143 @@
use std::sync::Arc;
use axum::{
extract::{Path, State},
http::StatusCode,
Json,
};
use uuid::Uuid;
use crate::routes::health::AppState;
use tools_common::types::*;
/// Handle GET /api/job/{id}
pub async fn job_status_handler(
State(state): State<Arc<AppState>>,
Path(id): Path<Uuid>,
) -> Result<Json<JobStatusResponse>, (StatusCode, Json<serde_json::Value>)> {
let mut conn = state.redis.get_multiplexed_async_connection().await.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({ "error": format!("Redis error: {}", e) })),
)
})?;
let job = crate::redis::job::JobRepository::get(&mut conn, id)
.await
.map_err(|_| {
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({ "error": "Job not found or expired" })),
)
})?;
let status_str = match &job.status {
JobStatus::Queued => "queued",
JobStatus::Processing { .. } => "processing",
JobStatus::Completed => "completed",
JobStatus::NeedsManualCrop => "needs_manual_crop",
JobStatus::Failed(_) => "failed",
};
let (progress, stage, message) = match &job.status {
JobStatus::Processing { stage, progress } => (*progress, stage.clone(), String::new()),
JobStatus::Failed(msg) => (0, String::new(), msg.clone()),
JobStatus::Completed => (100, "complete".to_string(), "Processing complete".to_string()),
JobStatus::Queued => (0, "queued".to_string(), "Waiting in queue".to_string()),
JobStatus::NeedsManualCrop => {
(0, "manual_crop".to_string(), "Manual crop needed".to_string())
}
};
let result = if job.status == JobStatus::Completed {
let file_name = job
.result_path
.as_ref()
.and_then(|p| std::path::Path::new(p).file_name())
.and_then(|n| n.to_str())
.unwrap_or("result")
.to_string();
Some(ResultInfo {
download_url: format!("/api/download/{}", job.id),
file_size: job.file_size,
file_name,
preview_url: Some(format!("/api/job/{}/preview", job.id)),
})
} else {
None
};
let error = match &job.status {
JobStatus::Failed(msg) => Some(msg.clone()),
_ => None,
};
Ok(Json(JobStatusResponse {
job_id: job.id,
status: status_str.to_string(),
tool: job.tool.as_str().to_string(),
progress,
stage,
message,
result,
created_at: job.created_at,
error,
}))
}
/// Handle GET /api/job/{id}/preview
pub async fn job_preview_handler(
State(state): State<Arc<AppState>>,
Path(id): Path<Uuid>,
) -> Result<(StatusCode, [(String, String); 2], Vec<u8>), (StatusCode, Json<serde_json::Value>)> {
let mut conn = state.redis.get_multiplexed_async_connection().await.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({ "error": format!("Redis error: {}", e) })),
)
})?;
let job = crate::redis::job::JobRepository::get(&mut conn, id)
.await
.map_err(|_| {
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({ "error": "Job not found or expired" })),
)
})?;
let result_path = job.result_path.ok_or_else(|| {
(
StatusCode::BAD_REQUEST,
Json(serde_json::json!({ "error": "No result available yet" })),
)
})?;
let data = tokio::fs::read(&result_path).await.map_err(|e| {
(
StatusCode::NOT_FOUND,
Json(serde_json::json!({ "error": format!("File not found: {}", e) })),
)
})?;
let ext = result_path.rsplit('.').next().unwrap_or("bin").to_string();
let content_type = match ext.as_str() {
"jpg" | "jpeg" => "image/jpeg",
"png" => "image/png",
"webp" => "image/webp",
"pdf" => "application/pdf",
_ => "application/octet-stream",
};
Ok((
StatusCode::OK,
[
("Content-Type".to_string(), content_type.to_string()),
(
"Cache-Control".to_string(),
"private, max-age=300".to_string(),
),
],
data,
))
}
@@ -0,0 +1,5 @@
pub mod download;
pub mod health;
pub mod job;
pub mod upload;
pub mod ws;
@@ -0,0 +1,261 @@
use std::sync::Arc;
use axum::{
extract::{Multipart, State},
http::StatusCode,
Json,
};
use chrono::Utc;
use tokio::fs;
use uuid::Uuid;
use crate::routes::health::AppState;
use tools_common::error::UploadError;
use tools_common::types::*;
/// Handle POST /api/upload
pub async fn upload_handler(
State(state): State<Arc<AppState>>,
mut multipart: Multipart,
) -> Result<Json<UploadResponse>, (StatusCode, Json<serde_json::Value>)> {
let config = &state.config;
let max_size = config.max_file_size_bytes();
// Extract fields from multipart
let mut file_data: Option<(String, Vec<u8>)> = None;
let mut tool_str: Option<String> = None;
let mut options: serde_json::Value = serde_json::Value::Null;
while let Ok(Some(field)) = multipart.next_field().await {
let name = field.name().unwrap_or("").to_string();
match name.as_str() {
"file" => {
let filename = field.file_name().unwrap_or("unknown").to_string();
let data = field.bytes().await.map_err(|e| {
(
StatusCode::BAD_REQUEST,
Json(serde_json::json!({
"error": "Failed to read file",
"detail": e.to_string()
})),
)
})?;
file_data = Some((filename, data.to_vec()));
}
"tool" => {
tool_str = Some(field.text().await.unwrap_or_default());
}
"options" => {
let text = field.text().await.unwrap_or_default();
if !text.is_empty() {
options = serde_json::from_str(&text).unwrap_or(serde_json::Value::Null);
}
}
_ => {}
}
}
// Validate fields
let (filename, data) = file_data.ok_or_else(|| {
(
StatusCode::BAD_REQUEST,
Json(serde_json::json!({ "error": "No file provided" })),
)
})?;
let tool_str = tool_str.ok_or_else(|| {
(
StatusCode::BAD_REQUEST,
Json(serde_json::json!({ "error": "No tool specified" })),
)
})?;
let tool = Tool::from_str(&tool_str).ok_or_else(|| {
(
StatusCode::BAD_REQUEST,
Json(serde_json::json!({ "error": format!("Unknown tool: {}", tool_str) })),
)
})?;
// Validate file size
let file_size = data.len() as u64;
if file_size > max_size {
return Err((
StatusCode::PAYLOAD_TOO_LARGE,
Json(serde_json::json!({
"error": format!("File too large: {} bytes (max {} bytes)", file_size, max_size)
})),
));
}
// Validate MIME type based on tool
let ext = filename.rsplit('.').next().unwrap_or("").to_lowercase();
validate_mime(&tool, &ext).map_err(|e| {
(
StatusCode::BAD_REQUEST,
Json(serde_json::json!({ "error": e.to_string() })),
)
})?;
// Verify magic bytes
if !verify_magic_bytes(&data, &ext) {
return Err((
StatusCode::BAD_REQUEST,
Json(serde_json::json!({ "error": "File content does not match extension" })),
));
}
// Create directories
let upload_dir = config.storage_path.join("upload");
fs::create_dir_all(&upload_dir).await.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({ "error": format!("Storage error: {}", e) })),
)
})?;
// Generate job ID and save file
let job_id = Uuid::new_v4();
let storage_filename = format!("{}.{}", job_id, ext);
let file_path = upload_dir.join(&storage_filename);
fs::write(&file_path, &data).await.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({ "error": format!("Failed to save file: {}", e) })),
)
})?;
// Create job record
let job = Job {
id: job_id,
tool: tool.clone(),
status: JobStatus::Queued,
file_path: file_path.to_string_lossy().to_string(),
result_path: None,
file_size,
options: options.clone(),
created_at: Utc::now(),
ttl_seconds: config.job_ttl_seconds,
};
// Save to Redis
{
let mut conn = state.redis.get_multiplexed_async_connection().await.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({ "error": format!("Redis error: {}", e) })),
)
})?;
crate::redis::job::JobRepository::create(&mut conn, &job)
.await
.map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({ "error": format!("Failed to create job: {}", e) })),
)
})?;
}
// Publish to NATS
crate::nats::publisher::NatsPublisher::publish_job(&state.nats, &tool, &job)
.await
.map_err(|e| {
tracing::error!("Failed to publish job to NATS: {}", e);
});
// Update metrics
state.metrics.increment_jobs_total(tool.as_str(), "queued");
// Return response
Ok(Json(UploadResponse {
job_id,
status: "queued".to_string(),
tool: tool_str,
ws_url: format!("/api/job/{}/ws", job_id),
created_at: job.created_at,
estimated_seconds: match tool {
Tool::Scan => 5,
_ => 3,
},
}))
}
fn validate_mime(tool: &Tool, ext: &str) -> Result<(), UploadError> {
let image_exts = ["jpg", "jpeg", "png", "webp", "heic", "bmp", "tiff", "tif"];
let pdf_exts = ["pdf"];
let video_exts = ["mp4", "webm", "avi", "mov", "mkv"];
let audio_exts = ["mp3", "wav", "flac", "aac", "ogg", "m4a"];
match tool {
Tool::Scan
| Tool::ImageCompress
| Tool::ImageResize
| Tool::ImageConvert
| Tool::RemoveBg => {
if !image_exts.contains(&ext) {
return Err(UploadError::InvalidMime(format!(
"Expected image file, got .{}",
ext
)));
}
}
Tool::PdfMerge | Tool::PdfSplit | Tool::PdfCompress | Tool::PdfToImages => {
if !pdf_exts.contains(&ext) {
return Err(UploadError::InvalidMime(format!(
"Expected PDF file, got .{}",
ext
)));
}
}
Tool::ImagesToPdf => {
if !image_exts.contains(&ext) {
return Err(UploadError::InvalidMime(format!(
"Expected image file, got .{}",
ext
)));
}
}
Tool::VideoCompress | Tool::VideoTrim | Tool::GifMaker => {
if !video_exts.contains(&ext) {
return Err(UploadError::InvalidMime(format!(
"Expected video file, got .{}",
ext
)));
}
}
Tool::AudioExtract => {
if !video_exts.contains(&ext) {
return Err(UploadError::InvalidMime(format!(
"Expected video file, got .{}",
ext
)));
}
}
Tool::AudioConvert => {
if !audio_exts.contains(&ext) {
return Err(UploadError::InvalidMime(format!(
"Expected audio file, got .{}",
ext
)));
}
}
}
Ok(())
}
fn verify_magic_bytes(data: &[u8], ext: &str) -> bool {
if data.is_empty() {
return false;
}
match ext {
"jpg" | "jpeg" => data.starts_with(&[0xFF, 0xD8, 0xFF]),
"png" => data.starts_with(&[0x89, 0x50, 0x4E, 0x47]),
"webp" => data.len() > 12 && &data[0..4] == b"RIFF" && &data[8..12] == b"WEBP",
"gif" => data.starts_with(b"GIF8"),
"bmp" => data.starts_with(b"BM"),
"pdf" => data.starts_with(b"%PDF"),
"mp4" => data.len() > 8 && (&data[4..8] == b"ftyp" || &data[4..8] == b"ftyp"),
"heic" => data.len() > 12 && &data[4..12] == b"ftypheic",
_ => true,
}
}
+141
View File
@@ -0,0 +1,141 @@
use std::sync::Arc;
use axum::{
extract::{
ws::{Message, WebSocket},
Path, State, WebSocketUpgrade,
},
response::IntoResponse,
};
use futures::{SinkExt, StreamExt};
use tokio::sync::mpsc;
use uuid::Uuid;
use crate::routes::health::AppState;
use tools_common::types::{JobProgress, JobStatus};
/// Handle WebSocket upgrade at /api/job/{id}/ws.
pub async fn ws_handler(
ws: WebSocketUpgrade,
State(state): State<Arc<AppState>>,
Path(job_id): Path<Uuid>,
) -> impl IntoResponse {
ws.on_upgrade(move |socket| handle_ws(socket, state, job_id))
}
async fn handle_ws(ws: WebSocket, state: Arc<AppState>, job_id: Uuid) {
let (mut sender, mut receiver) = ws.split();
// Subscribe to NATS progress updates
let nats = state.nats.clone();
let subject = format!("tools.*.progress.{}", job_id);
let mut subscriber = match nats.subscribe(subject).await {
Ok(sub) => sub,
Err(e) => {
tracing::error!("Failed to subscribe to NATS: {}", e);
let _ = sender
.send(Message::Text(
serde_json::json!({
"type": "error",
"job_id": job_id,
"status": "failed",
"error": format!("Connection error: {}", e)
})
.to_string()
.into(),
))
.await;
return;
}
};
// Send initial status from Redis
if let Ok(mut conn) = state.redis.get_multiplexed_async_connection().await {
if let Ok(job) = crate::redis::job::JobRepository::get(&mut conn, job_id).await {
let init_msg = serde_json::json!({
"type": "status",
"job_id": job_id,
"status": match &job.status {
JobStatus::Queued => "queued",
JobStatus::Processing { .. } => "processing",
JobStatus::Completed => "completed",
JobStatus::NeedsManualCrop => "needs_manual_crop",
JobStatus::Failed(_) => "failed",
},
"progress": match &job.status {
JobStatus::Processing { progress, .. } => *progress,
JobStatus::Completed => 100,
_ => 0,
},
});
let _ = sender
.send(Message::Text(init_msg.to_string().into()))
.await;
}
}
// Channel for NATS messages
let (tx, mut rx) = mpsc::channel::<String>(32);
// Spawn NATS listener
let tx_clone = tx.clone();
let nats_listener = tokio::spawn(async move {
loop {
tokio::select! {
msg = subscriber.next() => {
match msg {
Some(nats_msg) => {
if let Ok(progress) = serde_json::from_slice::<JobProgress>(&nats_msg.payload) {
let json = serde_json::json!({
"type": "progress",
"job_id": progress.job_id,
"status": match &progress.status {
JobStatus::Queued => "queued",
JobStatus::Processing { .. } => "processing",
JobStatus::Completed => "completed",
JobStatus::NeedsManualCrop => "needs_manual_crop",
JobStatus::Failed(_) => "failed",
},
"progress": progress.progress,
"stage": progress.stage,
"message": progress.message,
});
let _ = tx_clone.send(json.to_string()).await;
}
}
None => break,
}
}
_ = tokio::time::sleep(tokio::time::Duration::from_secs(30)) => {
// Keepalive ping
let _ = tx_clone.send(serde_json::json!({"type": "ping"}).to_string()).await;
}
}
}
});
// Forward messages from channel to WebSocket
let ws_sender = tokio::spawn(async move {
while let Some(msg) = rx.recv().await {
if sender.send(Message::Text(msg.into())).await.is_err() {
break;
}
}
});
// Listen for client close
let ws_receiver = tokio::spawn(async move {
while let Some(Ok(_)) = receiver.next().await {
// Client messages ignored (we only forward server→client)
}
});
// Wait for either task to complete (connection closed)
tokio::select! {
_ = ws_sender => {},
_ = ws_receiver => {},
}
// Cancel NATS listener
nats_listener.abort();
}
+2
View File
@@ -0,0 +1,2 @@
[toolchain]
channel = "1.85"
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "tools-wasm"
version.workspace = true
edition.workspace = true
license.workspace = true
[lib]
crate-type = ["cdylib"]
[dependencies]
wasm-bindgen = "0.2"
image.workspace = true
console_error_panic_hook = "0.1"
serde.workspace = true
serde_json.workspace = true
# Skip wasm crate from default cargo check
# Full build requires: wasm-pack build --target web
+18
View File
@@ -0,0 +1,18 @@
use wasm_bindgen::prelude::*;
/// Placeholder for WASM image processing.
/// Full implementation in Phase 2.2.
#[wasm_bindgen]
pub fn greet() -> String {
"tools-wasm: ready".to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_greet() {
assert_eq!(greet(), "tools-wasm: ready");
}
}
+32
View File
@@ -0,0 +1,32 @@
[package]
name = "tools-workers"
version.workspace = true
edition.workspace = true
license.workspace = true
[dependencies]
tools-common = { path = "../common" }
tokio.workspace = true
serde.workspace = true
serde_json.workspace = true
uuid.workspace = true
chrono.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
async-nats.workspace = true
redis.workspace = true
thiserror.workspace = true
anyhow.workspace = true
image.workspace = true
imageproc.workspace = true
nalgebra = "0.32"
lopdf.workspace = true
rayon = "1"
futures = "0.3"
async-trait = "0.1"
leptess = { version = "0.14", optional = true }
[features]
default = []
tesseract = ["leptess"]
@@ -0,0 +1,2 @@
// Audio processing module.
// TODO: Phase 4 - implement convert, trim
+33
View File
@@ -0,0 +1,33 @@
use std::path::PathBuf;
/// Worker configuration loaded from environment variables.
#[derive(Debug, Clone)]
pub struct WorkerConfig {
pub nats_url: String,
pub redis_url: String,
pub storage_path: PathBuf,
pub concurrency: u32,
pub job_ttl_seconds: u64,
pub rust_log: String,
}
impl WorkerConfig {
pub fn from_env() -> Self {
Self {
nats_url: env_or_default("NATS_URL", "nats://localhost:4222"),
redis_url: env_or_default("REDIS_URL", "redis://localhost:6379"),
storage_path: PathBuf::from(env_or_default("STORAGE_PATH", "/data/tools")),
concurrency: env_or_default("TOOLS_WORKER_CONCURRENCY", "4")
.parse()
.unwrap_or(4),
job_ttl_seconds: env_or_default("JOB_TTL_SECONDS", "3600")
.parse()
.unwrap_or(3600),
rust_log: env_or_default("RUST_LOG", "info"),
}
}
}
fn env_or_default(key: &str, default: &str) -> String {
std::env::var(key).unwrap_or_else(|_| default.to_string())
}
@@ -0,0 +1,15 @@
use crate::config::WorkerConfig;
use tools_common::types::Job;
/// Process an image tool job.
pub async fn process_job(
job: Job,
_redis: &redis::Client,
_config: &WorkerConfig,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
tracing::info!(job_id = %job.id, tool = %job.tool.as_str(), "Processing image job (stub)");
// TODO: Phase 2.2 - implement actual image processing
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
tracing::info!(job_id = %job.id, "Image job completed");
Ok(())
}
+39
View File
@@ -0,0 +1,39 @@
mod config;
mod image;
mod nats;
mod pdf;
mod scanner;
mod scheduler;
mod video;
mod audio;
use tracing_subscriber::EnvFilter;
#[tokio::main]
async fn main() {
let config = config::WorkerConfig::from_env();
tracing_subscriber::fmt()
.with_env_filter(EnvFilter::new(&config.rust_log))
.init();
tracing::info!("Starting tools-workers...");
// Connect to NATS
let nats = nats::consumer::JobConsumer::connect(&config.nats_url)
.await
.expect("Failed to connect to NATS");
tracing::info!("Connected to NATS at {}", config.nats_url);
// Connect to Redis
let redis = nats::consumer::JobConsumer::connect_redis(&config.redis_url)
.await
.expect("Failed to connect to Redis");
tracing::info!("Connected to Redis at {}", config.redis_url);
// Start NATS consumers (blocks forever)
tracing::info!("Starting job consumers...");
if let Err(e) = nats::consumer::JobConsumer::start(&nats, &redis, &config).await {
tracing::error!("Consumer error: {}", e);
}
}
@@ -0,0 +1,186 @@
use async_nats::Client;
use futures::StreamExt;
use redis::AsyncCommands;
use uuid::Uuid;
use tools_common::types::{Job, JobStatus};
use crate::config::WorkerConfig;
/// NATS consumer setup and management.
pub struct JobConsumer;
impl JobConsumer {
/// Connect to NATS.
pub async fn connect(url: &str) -> Result<Client, Box<dyn std::error::Error + Send + Sync>> {
Ok(async_nats::connect(url).await?)
}
/// Connect to Redis.
pub async fn connect_redis(
url: &str,
) -> Result<redis::Client, Box<dyn std::error::Error + Send + Sync>> {
Ok(redis::Client::open(url)?)
}
/// Start consuming job messages from NATS for all tool groups.
pub async fn start(
nats: &Client,
redis: &redis::Client,
config: &WorkerConfig,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Subscribe to scan jobs
let scan_sub = nats
.queue_subscribe("tools.scan.jobs.>", "scan-workers".to_string())
.await?;
tracing::info!("Subscribed to tools.scan.jobs.>");
// Subscribe to image jobs
let image_sub = nats
.queue_subscribe("tools.image.jobs.>", "image-workers".to_string())
.await?;
tracing::info!("Subscribed to tools.image.jobs.>");
// Subscribe to pdf jobs
let pdf_sub = nats
.queue_subscribe("tools.pdf.jobs.>", "pdf-workers".to_string())
.await?;
tracing::info!("Subscribed to tools.pdf.jobs.>");
// Subscribe to cleanup scheduler
let cleanup_sub = nats
.subscribe("tools.scheduler.cleanup".to_string())
.await?;
tracing::info!("Subscribed to tools.scheduler.cleanup");
let redis_clone = redis.clone();
let config_clone = config.clone();
// Process messages concurrently
tokio::select! {
_ = Self::process_subscription(scan_sub, redis.clone(), config.clone()) => {},
_ = Self::process_subscription(image_sub, redis.clone(), config.clone()) => {},
_ = Self::process_subscription(pdf_sub, redis.clone(), config.clone()) => {},
_ = Self::process_cleanup(cleanup_sub, config_clone) => {},
}
Ok(())
}
/// Process messages from a NATS subscription.
async fn process_subscription(
mut sub: async_nats::Subscriber,
redis: redis::Client,
config: WorkerConfig,
) {
while let Some(msg) = sub.next().await {
if let Ok(job) = serde_json::from_slice::<Job>(&msg.payload) {
let redis = redis.clone();
let config = config.clone();
tokio::spawn(async move {
let tool = job.tool.clone();
tracing::info!(
job_id = %job.id,
tool = %tool.as_str(),
"Received job"
);
match Self::dispatch_job(tool, job, &redis, &config).await {
Ok(()) => tracing::info!("Job completed successfully"),
Err(e) => tracing::error!("Job failed: {}", e),
}
});
}
}
}
/// Process cleanup scheduler messages.
async fn process_cleanup(mut sub: async_nats::Subscriber, config: WorkerConfig) {
while let Some(msg) = sub.next().await {
tracing::info!("Running cleanup cycle");
let redis_url = config.redis_url.clone();
match redis::Client::open(redis_url.as_str()) {
Ok(client) => {
match crate::scheduler::cleanup::CleanupScheduler::run(
&config.storage_path,
&client,
config.job_ttl_seconds,
)
.await
{
Ok(result) => {
tracing::info!(
"Cleanup: {} files deleted, {} bytes freed",
result.files_deleted,
result.bytes_freed
);
}
Err(e) => {
tracing::error!("Cleanup failed: {}", e);
}
}
}
Err(e) => {
tracing::error!("Failed to create Redis client for cleanup: {}", e);
}
}
// Consume the message (no ack for core NATS)
let _ = msg;
}
}
/// Dispatch a job to the appropriate handler based on tool type.
async fn dispatch_job(
tool: tools_common::types::Tool,
job: Job,
redis: &redis::Client,
config: &WorkerConfig,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
match tool {
tools_common::types::Tool::Scan => {
crate::scanner::process_job(job, redis, config).await
}
tools_common::types::Tool::ImageCompress
| tools_common::types::Tool::ImageResize
| tools_common::types::Tool::ImageConvert
| tools_common::types::Tool::RemoveBg => {
crate::image::process_job(job, redis, config).await
}
tools_common::types::Tool::PdfMerge
| tools_common::types::Tool::PdfSplit
| tools_common::types::Tool::ImagesToPdf
| tools_common::types::Tool::PdfCompress
| tools_common::types::Tool::PdfToImages => {
crate::pdf::process_job(job, redis, config).await
}
_ => {
tracing::warn!(tool = %tool.as_str(), "Tool handler not yet implemented");
Ok(())
}
}
}
/// Update job result in Redis after processing.
pub async fn update_job_result(
conn: &mut impl AsyncCommands,
job_id: Uuid,
result_path: &str,
ttl_seconds: u64,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let key = format!("job:{}", job_id);
let json: String = conn
.get(&key)
.await
.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?;
let mut job: Job = serde_json::from_str(&json)?;
job.status = JobStatus::Completed;
job.result_path = Some(result_path.to_string());
let updated = serde_json::to_string(&job)?;
let _: () = conn
.set_ex(key, updated, ttl_seconds)
.await
.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?;
Ok(())
}
}
@@ -0,0 +1,2 @@
pub mod consumer;
pub mod progress;
@@ -0,0 +1,82 @@
use redis::AsyncCommands;
use uuid::Uuid;
use tools_common::types::{JobStatus, Tool};
/// Reports progress from worker to NATS and Redis.
pub struct ProgressReporter {
redis: redis::Client,
nats: async_nats::Client,
job_id: Uuid,
tool: Tool,
}
impl ProgressReporter {
pub fn new(redis: redis::Client, nats: async_nats::Client, job_id: Uuid, tool: Tool) -> Self {
Self {
redis,
nats,
job_id,
tool,
}
}
/// Report progress: updates Redis and publishes to NATS.
pub async fn report(
&self,
status: JobStatus,
stage: &str,
progress: u8,
message: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Update Redis
if let Ok(mut conn) = self.redis.get_multiplexed_async_connection().await {
let key = format!("job:{}", self.job_id);
if let Ok(json) = conn.get::<_, String>(&key).await {
if let Ok(mut job) = serde_json::from_str::<tools_common::types::Job>(&json) {
job.status = status.clone();
let updated = serde_json::to_string(&job).unwrap_or(json);
let _: Result<(), _> = conn.set_ex(key, updated, job.ttl_seconds).await;
}
}
}
// Publish to NATS
let progress_msg = tools_common::types::JobProgress {
job_id: self.job_id,
status,
stage: stage.to_string(),
progress,
message: message.to_string(),
};
let subject = format!("tools.{}.progress.{}", self.tool.subject_prefix(), self.job_id);
if let Ok(payload) = serde_json::to_vec(&progress_msg) {
let _ = self.nats.publish(subject, payload.into()).await;
}
tracing::debug!(
job_id = %self.job_id,
stage = %stage,
progress = %progress,
"Progress update"
);
Ok(())
}
pub fn job_id(&self) -> Uuid {
self.job_id
}
}
impl Clone for ProgressReporter {
fn clone(&self) -> Self {
Self {
redis: self.redis.clone(),
nats: self.nats.clone(),
job_id: self.job_id,
tool: self.tool.clone(),
}
}
}
+15
View File
@@ -0,0 +1,15 @@
use crate::config::WorkerConfig;
use tools_common::types::Job;
/// Process a PDF tool job.
pub async fn process_job(
job: Job,
_redis: &redis::Client,
_config: &WorkerConfig,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
tracing::info!(job_id = %job.id, tool = %job.tool.as_str(), "Processing PDF job (stub)");
// TODO: Phase 3 - implement actual PDF processing
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
tracing::info!(job_id = %job.id, "PDF job completed");
Ok(())
}
@@ -0,0 +1,246 @@
use image::{GrayImage, Luma};
/// Apply Sauvola local threshold for clean black-and-white output.
///
/// Sauvola: T(x,y) = m(x,y) * [1 + k * (s(x,y)/R - 1)]
/// where m = local mean, s = local std dev, R = 128, k = 0.2
pub fn sauvola_threshold(img: &GrayImage, window_size: u32, k: f64) -> GrayImage {
let (w, h) = (img.width(), img.height());
let half_win = (window_size / 2) as i32;
let mut output = GrayImage::new(w, h);
// Integral images for O(1) mean and variance computation
let integral = compute_integral_image(img);
let integral_sq = compute_integral_image_sq(img);
for y in 0..h {
for x in 0..w {
let (mean, variance) = local_stats(
&integral,
&integral_sq,
x as i32,
y as i32,
half_win,
w as i32,
h as i32,
);
let std_dev = variance.sqrt();
let threshold = mean * (1.0 + k * (std_dev / 128.0 - 1.0));
let pixel = img.get_pixel(x, y)[0] as f64;
output.put_pixel(x, y, Luma([if pixel > threshold { 255 } else { 0 }]));
}
}
output
}
/// Compute integral image for O(1) sum queries.
fn compute_integral_image(img: &GrayImage) -> Vec<u64> {
let (w, h) = (img.width() as usize, img.height() as usize);
let mut integral = vec![0u64; (w + 1) * (h + 1)];
for y in 0..h {
for x in 0..w {
let idx = (y + 1) * (w + 1) + (x + 1);
let pixel = img.get_pixel(x as u32, y as u32)[0] as u64;
integral[idx] = pixel
+ integral[(y + 1) * (w + 1) + x]
+ integral[y * (w + 1) + (x + 1)]
- integral[y * (w + 1) + x];
}
}
integral
}
/// Compute squared integral image for O(1) variance queries.
fn compute_integral_image_sq(img: &GrayImage) -> Vec<u64> {
let (w, h) = (img.width() as usize, img.height() as usize);
let mut integral = vec![0u64; (w + 1) * (h + 1)];
for y in 0..h {
for x in 0..w {
let idx = (y + 1) * (w + 1) + (x + 1);
let pixel = img.get_pixel(x as u32, y as u32)[0] as u64;
let pixel_sq = pixel * pixel;
integral[idx] = pixel_sq
+ integral[(y + 1) * (w + 1) + x]
+ integral[y * (w + 1) + (x + 1)]
- integral[y * (w + 1) + x];
}
}
integral
}
/// Compute local mean and variance for a window around (x, y) using integral images.
fn local_stats(
integral: &[u64],
integral_sq: &[u64],
x: i32,
y: i32,
half_win: i32,
w: i32,
h: i32,
) -> (f64, f64) {
let x1 = (x - half_win).max(0);
let y1 = (y - half_win).max(0);
let x2 = (x + half_win).min(w - 1);
let y2 = (y + half_win).min(h - 1);
let width = (w + 1) as usize;
let area = ((x2 - x1 + 1) * (y2 - y1 + 1)) as f64;
if area <= 0.0 {
return (0.0, 0.0);
}
// Sum from integral image
let idx_tl = (y1) as usize * width + (x1) as usize;
let idx_tr = (y1) as usize * width + (x2 + 1) as usize;
let idx_bl = (y2 + 1) as usize * width + (x1) as usize;
let idx_br = (y2 + 1) as usize * width + (x2 + 1) as usize;
let sum = integral[idx_br]
.wrapping_sub(integral[idx_tr])
.wrapping_sub(integral[idx_bl])
.wrapping_add(integral[idx_tl]);
// Sum of squares
let sum_sq = integral_sq[idx_br]
.wrapping_sub(integral_sq[idx_tr])
.wrapping_sub(integral_sq[idx_bl])
.wrapping_add(integral_sq[idx_tl]);
let mean = sum as f64 / area;
let variance = (sum_sq as f64 / area) - mean * mean;
(mean, variance.max(0.0))
}
/// Otsu global threshold (fallback for when Sauvola is too slow).
#[allow(dead_code)]
pub fn otsu_threshold(img: &GrayImage) -> GrayImage {
let (w, h) = (img.width(), img.height());
let total_pixels = w * h;
// Compute histogram
let mut hist = [0u32; 256];
for pixel in img.iter() {
hist[*pixel as usize] += 1;
}
// Normalize to probabilities
let mut prob = [0.0f64; 256];
for i in 0..256 {
prob[i] = hist[i] as f64 / total_pixels as f64;
}
// Find threshold that maximizes between-class variance
let mut best_threshold = 128u8;
let mut best_variance = 0.0f64;
for t in 1..255 {
let w0: f64 = prob[..t].iter().sum();
let w1: f64 = prob[t..].iter().sum();
if w0 < 1e-6 || w1 < 1e-6 {
continue;
}
let mut mean0 = 0.0f64;
let mut mean1 = 0.0f64;
for i in 0..t {
mean0 += i as f64 * prob[i] / w0;
}
for i in t..256 {
mean1 += i as f64 * prob[i] / w1;
}
let variance = w0 * w1 * (mean0 - mean1).powi(2);
if variance > best_variance {
best_variance = variance;
best_threshold = t as u8;
}
}
// Apply threshold
let mut output = GrayImage::new(w, h);
for y in 0..h {
for x in 0..w {
let pixel = img.get_pixel(x, y)[0];
output.put_pixel(x, y, Luma([if pixel > best_threshold { 255 } else { 0 }]));
}
}
output
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_sauvola_on_simple_image() {
// Create document-like image: white background with dark text lines
let mut img = GrayImage::new(100, 100);
// White background
for y in 0..100 {
for x in 0..100 {
img.put_pixel(x, y, Luma([220]));
}
}
// Dark text lines (simulated with thin dark rectangles)
for y in 0..100 {
for x in 0..100 {
// Alternate thin dark "text" lines
if y % 10 < 3 && x > 10 && x < 90 {
img.put_pixel(x, y, Luma([30]));
}
}
}
let result = sauvola_threshold(&img, 25, 0.2);
// Text line at y=1 should be black (0)
let text_pixel1 = result.get_pixel(50, 1)[0];
let text_pixel2 = result.get_pixel(50, 2)[0];
assert_eq!(text_pixel1, 0, "Text line at y=1 should be black (0), got {}", text_pixel1);
assert_eq!(text_pixel2, 0, "Text line at y=2 should be black (0), got {}", text_pixel2);
// Background at y=5 should be white (255)
let bg_pixel = result.get_pixel(50, 5)[0];
assert_eq!(bg_pixel, 255, "Background at y=5 should be white (255), got {}", bg_pixel);
}
#[test]
fn test_integral_image() {
let mut img = GrayImage::new(4, 4);
img.put_pixel(0, 0, Luma([1]));
img.put_pixel(1, 0, Luma([2]));
img.put_pixel(0, 1, Luma([3]));
img.put_pixel(1, 1, Luma([4]));
let integral = compute_integral_image(&img);
let width = 5; // (w+1)
// Sum of all 4 pixels at (2,2)
let sum = integral[2 * width + 2];
assert_eq!(sum, 1 + 2 + 3 + 4); // 10
}
#[test]
fn test_otsu_on_bimodal() {
// Create a bimodal image: half black, half white
let mut img = GrayImage::new(50, 50);
for y in 0..50 {
for x in 0..50 {
let val = if x < 25 { 30 } else { 200 };
img.put_pixel(x, y, Luma([val]));
}
}
let result = otsu_threshold(&img);
// Should threshold correctly at ~115
assert_eq!(result.get_pixel(10, 25)[0], 0); // dark side
assert_eq!(result.get_pixel(35, 25)[0], 255); // light side
}
}
@@ -0,0 +1,177 @@
use image::GrayImage;
use imageproc::contours::find_contours;
use tools_common::error::PipelineError;
/// Represents a detected corner point.
pub type CornerPoint = (f64, f64);
/// The fallback reason if corner detection fails.
pub enum FallbackReason {
NoContours,
NoRectangularContour,
TooSmall,
}
/// Find the 4 corners of the document from an edge image.
pub fn detect_corners(edges: &GrayImage) -> Result<[CornerPoint; 4], FallbackReason> {
let contours = find_contours::<u8>(edges);
if contours.is_empty() {
return Err(FallbackReason::NoContours);
}
// Convert contours to use i32 coordinates
let contour_points: Vec<Vec<(i32, i32)>> = contours
.iter()
.map(|c| c.points.iter().map(|p| (p.x as i32, p.y as i32)).collect())
.collect();
// Sort by area descending
let mut sorted: Vec<_> = contour_points.iter().collect();
sorted.sort_by(|a, b| {
contour_area_slice(b)
.partial_cmp(&contour_area_slice(a))
.unwrap_or(std::cmp::Ordering::Equal)
});
for points in sorted.iter().take(5) {
if let Some(corners) = approx_quadrilateral(points) {
let ordered = order_corners(&corners);
return Ok(ordered);
}
}
// Fallback: use bounding rect of largest contour
if let Some(largest) = sorted.first() {
let rect = bounding_rect_slice(largest);
let corners = vec![
(rect.0 as f64, rect.1 as f64),
(rect.2 as f64, rect.1 as f64),
(rect.2 as f64, rect.3 as f64),
(rect.0 as f64, rect.3 as f64),
];
return Ok(order_corners(&corners));
}
Err(FallbackReason::NoContours)
}
/// Compute the area of a contour using the Shoelace formula.
fn contour_area_slice(points: &[(i32, i32)]) -> f64 {
let n = points.len();
if n < 3 {
return 0.0;
}
let mut area = 0.0;
for i in 0..n {
let j = (i + 1) % n;
area += points[i].0 as f64 * points[j].1 as f64;
area -= points[j].0 as f64 * points[i].1 as f64;
}
area.abs() / 2.0
}
/// Approximate a contour to a quadrilateral.
fn approx_quadrilateral(points: &[(i32, i32)]) -> Option<Vec<CornerPoint>> {
let n = points.len();
if n < 4 {
return None;
}
let top = points.iter().min_by(|a, b| a.1.cmp(&b.1))?;
let bottom = points.iter().max_by(|a, b| a.1.cmp(&b.1))?;
let left = points.iter().min_by(|a, b| a.0.cmp(&b.0))?;
let right = points.iter().max_by(|a, b| a.0.cmp(&b.0))?;
Some(vec![
(left.0 as f64, left.1 as f64),
(right.0 as f64, top.1 as f64),
(right.0 as f64, bottom.1 as f64),
(left.0 as f64, bottom.1 as f64),
])
}
/// Order 4 corners: top-left, top-right, bottom-right, bottom-left.
fn order_corners(points: &[CornerPoint]) -> [CornerPoint; 4] {
let mut pts: Vec<CornerPoint> = points.to_vec();
let mut ordered = [(0.0, 0.0); 4];
if pts.len() >= 4 {
// Sort by position
// TL = min(x+y), BR = max(x+y)
pts.sort_by(|a, b| {
(a.0 + a.1)
.partial_cmp(&(b.0 + b.1))
.unwrap_or(std::cmp::Ordering::Equal)
});
ordered[0] = pts[0]; // TL
ordered[2] = pts[3]; // BR
// TR = max(x - y), BL = min(x - y)
pts.sort_by(|a, b| {
(a.0 - a.1)
.partial_cmp(&(b.0 - b.1))
.unwrap_or(std::cmp::Ordering::Equal)
});
ordered[1] = pts[3]; // TR
ordered[3] = pts[0]; // BL
}
ordered
}
/// Compute bounding rectangle: (left, top, right, bottom).
fn bounding_rect_slice(points: &[(i32, i32)]) -> (i32, i32, i32, i32) {
let left = points.iter().map(|p| p.0).min().unwrap_or(0);
let top = points.iter().map(|p| p.1).min().unwrap_or(0);
let right = points.iter().map(|p| p.0).max().unwrap_or(0);
let bottom = points.iter().map(|p| p.1).max().unwrap_or(0);
(left, top, right, bottom)
}
/// Detect corners with fallback: full resolution, then half, then error.
pub fn detect_corners_with_fallback(
edges: &GrayImage,
) -> Result<[CornerPoint; 4], PipelineError> {
// Attempt 1: Full resolution
if let Ok(corners) = detect_corners(edges) {
return Ok(corners);
}
// Attempt 2: Half resolution
let (w, h) = (edges.width() / 2, edges.height() / 2);
if w > 10 && h > 10 {
let half = image::imageops::resize(
edges,
w,
h,
image::imageops::FilterType::Lanczos3,
);
if let Ok(corners) = detect_corners(&half) {
return Ok(corners.map(|(x, y)| (x * 2.0, y * 2.0)));
}
}
Err(PipelineError::CornerDetection(
"Could not detect document corners automatically".to_string(),
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_contour_area_slice() {
let points = vec![(0, 0), (100, 0), (100, 100), (0, 100)];
let area = contour_area_slice(&points);
assert!((area - 10000.0).abs() < 1.0);
}
#[test]
fn test_bounding_rect_slice() {
let points = vec![(10, 20), (100, 30), (90, 150), (5, 140)];
let rect = bounding_rect_slice(&points);
assert_eq!(rect, (5, 20, 100, 150));
}
}
@@ -0,0 +1,197 @@
use image::{GrayImage, Luma};
use image::imageops;
/// Detect and correct small rotation (<5°) of text lines using Hough transform.
pub fn deskew(img: &GrayImage) -> GrayImage {
let lines = hough_lines(img, 10, 50);
if lines.is_empty() {
return img.clone();
}
// Compute median angle of all detected lines
let angles: Vec<f64> = lines
.iter()
.map(|line| line.angle_deg())
.filter(|a| a.abs() < 45.0) // Skip vertical lines
.collect();
if angles.is_empty() {
return img.clone();
}
let median_angle = median(&angles);
// Skip if angle is very small (<0.5°)
if median_angle.abs() < 0.5 {
return img.clone();
}
// Rotate image
rotate_image(img, median_angle)
}
/// Represents a line detected by Hough transform.
#[derive(Debug, Clone)]
struct HoughLine {
rho: f64,
theta: f64,
}
impl HoughLine {
fn angle_deg(&self) -> f64 {
self.theta.to_degrees() - 90.0
}
}
/// Simple Hough line detection.
fn hough_lines(img: &GrayImage, threshold: u32, _max_lines: usize) -> Vec<HoughLine> {
let (w, h) = (img.width() as i32, img.height() as i32);
let max_rho = ((w * w + h * h) as f64).sqrt().ceil() as i32;
let theta_step = 1.0_f64.to_radians();
let num_thetas = 180;
// Accumulator
let mut accumulator =
vec![vec![0u32; (2 * max_rho + 1) as usize]; num_thetas];
// Vote
for y in 0..h {
for x in 0..w {
if img.get_pixel(x as u32, y as u32)[0] > 128 {
for t_idx in 0..num_thetas {
let theta = t_idx as f64 * theta_step;
let rho = (x as f64 * theta.cos() + y as f64 * theta.sin()).round() as i32;
let rho_idx = rho + max_rho;
if rho_idx >= 0 && (rho_idx as usize) < accumulator[t_idx].len() {
accumulator[t_idx][rho_idx as usize] += 1;
}
}
}
}
}
// Find local maxima above threshold
let mut lines = Vec::new();
for t_idx in 0..num_thetas {
let theta = t_idx as f64 * theta_step;
for (r_idx, &count) in accumulator[t_idx].iter().enumerate() {
if count > threshold {
let rho = r_idx as i32 - max_rho;
lines.push(HoughLine {
rho: rho as f64,
theta,
});
}
}
}
// Sort by votes (descending) and take top N
lines.sort_by(|a, b| {
let a_idx = (a.theta / theta_step).round() as usize;
let b_idx = (b.theta / theta_step).round() as usize;
let a_rho_idx = (a.rho + max_rho as f64).round() as usize;
let b_rho_idx = (b.rho + max_rho as f64).round() as usize;
let a_count = accumulator[a_idx.min(num_thetas - 1)][a_rho_idx.min(accumulator[0].len() - 1)];
let b_count = accumulator[b_idx.min(num_thetas - 1)][b_rho_idx.min(accumulator[0].len() - 1)];
b_count.cmp(&a_count)
});
lines.truncate(100);
lines
}
/// Compute median of a sorted slice of f64 values.
fn median(values: &[f64]) -> f64 {
if values.is_empty() {
return 0.0;
}
let mut sorted = values.to_vec();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let mid = sorted.len() / 2;
if sorted.len() % 2 == 0 {
(sorted[mid - 1] + sorted[mid]) / 2.0
} else {
sorted[mid]
}
}
/// Rotate an image by the given angle in degrees.
fn rotate_image(img: &GrayImage, angle_deg: f64) -> GrayImage {
let angle_rad = angle_deg.to_radians();
let (w, h) = (img.width(), img.height());
// Compute new image dimensions to fit the rotated content
let cos = angle_rad.cos().abs();
let sin = angle_rad.sin().abs();
let new_w = (w as f64 * cos + h as f64 * sin).ceil() as u32;
let new_h = (w as f64 * sin + h as f64 * cos).ceil() as u32;
let new_w = new_w.max(1);
let new_h = new_h.max(1);
let mut output = GrayImage::new(new_w, new_h);
let cx = w as f64 / 2.0;
let cy = h as f64 / 2.0;
let new_cx = new_w as f64 / 2.0;
let new_cy = new_h as f64 / 2.0;
// Backward mapping
for out_y in 0..new_h {
for out_x in 0..new_w {
// Translate to origin, rotate, translate back
let dx = out_x as f64 - new_cx;
let dy = out_y as f64 - new_cy;
let src_x = dx * cos + dy * sin + cx;
let src_y = -dx * sin + dy * cos + cy;
if src_x >= 0.0 && src_x < w as f64 - 1.0 && src_y >= 0.0 && src_y < h as f64 - 1.0 {
// Bilinear interpolation
let x0 = src_x.floor() as u32;
let y0 = src_y.floor() as u32;
let x1 = (x0 + 1).min(w - 1);
let y1 = (y0 + 1).min(h - 1);
let fx = src_x - x0 as f64;
let fy = src_y - y0 as f64;
let p00 = img.get_pixel(x0, y0)[0] as f64;
let p10 = img.get_pixel(x1, y0)[0] as f64;
let p01 = img.get_pixel(x0, y1)[0] as f64;
let p11 = img.get_pixel(x1, y1)[0] as f64;
let val = p00 * (1.0 - fx) * (1.0 - fy)
+ p10 * fx * (1.0 - fy)
+ p01 * (1.0 - fx) * fy
+ p11 * fx * fy;
output.put_pixel(out_x, out_y, Luma([val.round().clamp(0.0, 255.0) as u8]));
} else {
output.put_pixel(out_x, out_y, Luma([255])); // White padding
}
}
}
output
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_median_odd() {
let v = vec![1.0, 3.0, 5.0];
assert!((median(&v) - 3.0).abs() < 0.001);
}
#[test]
fn test_median_even() {
let v = vec![1.0, 2.0, 3.0, 4.0];
assert!((median(&v) - 2.5).abs() < 0.001);
}
#[test]
fn test_empty() {
assert!((median(&[]) - 0.0).abs() < 0.001);
}
}
@@ -0,0 +1,77 @@
use image::{GrayImage};
use imageproc::edges::canny;
use imageproc::filter::gaussian_blur_f32;
use imageproc::distance_transform::Norm;
use imageproc::morphology::close;
use tools_common::error::PipelineError;
/// Detect edges using Canny algorithm with adaptive threshold.
pub fn detect_edges(img: &GrayImage) -> Result<GrayImage, PipelineError> {
// 1. Gaussian blur for noise reduction
let blurred = gaussian_blur_f32(img, 3.0);
// 2. First attempt: Canny with standard thresholds
let edges = canny(&blurred, 50.0, 150.0);
// 3. Morphological close to connect broken edges
let closed = close(&edges, Norm::L1, 5);
// 4. Check edge coverage
let edge_count = count_non_zero(&closed);
let total_pixels = (closed.width() * closed.height()) as u32;
// If too few edges (<1%), retry with lower thresholds
if edge_count < total_pixels / 100 {
let edges2 = canny(&blurred, 20.0, 80.0);
let closed2 = close(&edges2, Norm::L1, 5);
let edge_count2 = count_non_zero(&closed2);
if edge_count2 < total_pixels / 200 {
return Err(PipelineError::EdgeDetection(
"Too few edges detected even with low threshold".to_string(),
));
}
return Ok(closed2);
}
Ok(closed)
}
/// Count non-zero (white) pixels in a binary image.
fn count_non_zero(img: &GrayImage) -> u32 {
let mut count = 0u32;
for pixel in img.iter() {
if *pixel > 0 {
count += 1;
}
}
count
}
#[cfg(test)]
mod tests {
use super::*;
use image::Luma;
#[test]
fn test_edge_detection_on_simple_image() {
let mut img = GrayImage::new(200, 200);
for y in 30..170 {
for x in 30..170 {
img.put_pixel(x, y, Luma([255]));
}
}
let result = detect_edges(&img);
assert!(result.is_ok());
let edges = result.unwrap();
assert!(count_non_zero(&edges) > 0);
}
#[test]
fn test_empty_image_returns_error() {
let img = GrayImage::new(100, 100);
let result = detect_edges(&img);
assert!(result.is_err());
}
}
@@ -0,0 +1,134 @@
use image::{GrayImage, Luma};
use imageproc::filter::gaussian_blur_f32;
/// Apply final sharpening and contrast optimization.
pub fn enhance_final(img: &GrayImage) -> GrayImage {
let sharpened = unsharp_mask(img, 1.0, 1.0);
adjust_contrast(&sharpened, 1.2)
}
/// Unsharp mask: add high-frequency detail back to the image.
/// result = img + amount * (img - blurred)
pub fn unsharp_mask(img: &GrayImage, sigma: f64, amount: f64) -> GrayImage {
let (w, h) = (img.width(), img.height());
let blurred = gaussian_blur_f32(img, sigma as f32);
let mut output = GrayImage::new(w, h);
for y in 0..h {
for x in 0..w {
let orig = img.get_pixel(x, y)[0] as f64;
let blur = blurred.get_pixel(x, y)[0] as f64;
let mask = orig - blur;
let result = (orig + amount * mask).clamp(0.0, 255.0) as u8;
output.put_pixel(x, y, Luma([result]));
}
}
output
}
/// Adjust contrast by scaling pixel values around the mean.
pub fn adjust_contrast(img: &GrayImage, factor: f64) -> GrayImage {
let (w, h) = (img.width(), img.height());
let mean = mean_value(img);
let mut output = GrayImage::new(w, h);
for y in 0..h {
for x in 0..w {
let pixel = img.get_pixel(x, y)[0] as f64;
let adjusted = ((pixel - mean) * factor + mean).clamp(0.0, 255.0) as u8;
output.put_pixel(x, y, Luma([adjusted]));
}
}
output
}
/// Remove salt-and-pepper noise using a median-like filter.
#[allow(dead_code)]
pub fn remove_noise(img: &GrayImage, threshold: u8) -> GrayImage {
let (w, h) = (img.width(), img.height());
let mut output = GrayImage::new(w, h);
for y in 1..h - 1 {
for x in 1..w - 1 {
let center = img.get_pixel(x, y)[0];
// Check if pixel is significantly different from neighbors
let mut neighbors = Vec::new();
for dy in -1i32..=1 {
for dx in -1i32..=1 {
if dx == 0 && dy == 0 {
continue;
}
neighbors.push(
img.get_pixel((x as i32 + dx) as u32, (y as i32 + dy) as u32)[0],
);
}
}
let min = *neighbors.iter().min().unwrap_or(&0);
let max = *neighbors.iter().max().unwrap_or(&255);
if (center as i16 - min as i16).abs() > threshold as i16
|| (center as i16 - max as i16).abs() > threshold as i16
{
// Replace with median
neighbors.sort();
output.put_pixel(x, y, Luma([neighbors[neighbors.len() / 2]]));
} else {
output.put_pixel(x, y, Luma([center]));
}
}
}
// Copy edges
for x in 0..w {
output.put_pixel(x, 0, *img.get_pixel(x, 0));
output.put_pixel(x, h - 1, *img.get_pixel(x, h - 1));
}
for y in 0..h {
output.put_pixel(0, y, *img.get_pixel(0, y));
output.put_pixel(w - 1, y, *img.get_pixel(w - 1, y));
}
output
}
/// Compute mean pixel value.
fn mean_value(img: &GrayImage) -> f64 {
let sum: u64 = img.iter().map(|&p| p as u64).sum();
let count = img.width() as u64 * img.height() as u64;
if count > 0 {
sum as f64 / count as f64
} else {
128.0
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_unsharp_mask_no_change() {
// Uniform image should remain unchanged
let img = GrayImage::from_pixel(50, 50, Luma([128]));
let result = unsharp_mask(&img, 1.0, 0.0);
assert_eq!(result.get_pixel(25, 25)[0], 128);
}
#[test]
fn test_contrast_increase() {
let mut img = GrayImage::new(10, 10);
img.put_pixel(0, 0, Luma([100]));
img.put_pixel(1, 0, Luma([200]));
let result = adjust_contrast(&img, 2.0);
// With factor > 1, contrast increases
let diff_orig = (200 - 100) as f64;
let diff_result = (result.get_pixel(1, 0)[0] as f64) - (result.get_pixel(0, 0)[0] as f64);
// The difference after contrast adjustment should be greater than original
assert!(
diff_result.abs() > diff_orig.abs() * 0.5,
"diff_orig={}, diff_result={}",
diff_orig,
diff_result
);
}
}
@@ -0,0 +1,80 @@
pub mod binarize;
pub mod corners;
pub mod deskew;
pub mod edge;
pub mod enhance;
pub mod ocr;
pub mod pdf;
pub mod pipeline;
pub mod preprocess;
pub mod shadow;
pub mod warp;
use crate::config::WorkerConfig;
use crate::nats::progress::ProgressReporter;
use tools_common::types::{Job, JobStatus, Tool};
/// Process a scan job through the full pipeline.
pub async fn process_job(
job: Job,
redis: &redis::Client,
config: &WorkerConfig,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
tracing::info!(job_id = %job.id, "Processing scan job");
let nats = async_nats::connect(&config.nats_url).await?;
let progress = ProgressReporter::new(redis.clone(), nats, job.id, Tool::Scan);
progress
.report(
JobStatus::Processing {
stage: "preprocess".to_string(),
progress: 5,
},
"preprocess",
5,
"Memproses gambar...",
)
.await?;
let result = pipeline::process(&job, config, &progress).await;
match result {
Ok(scan_result) => {
progress
.report(JobStatus::Completed, "complete", 100, "Scan selesai")
.await?;
let mut conn = redis.get_multiplexed_async_connection().await?;
crate::nats::consumer::JobConsumer::update_job_result(
&mut conn,
job.id,
&scan_result.output_path,
job.ttl_seconds,
)
.await?;
tracing::info!(
job_id = %job.id,
output = %scan_result.output_path,
duration_ms = %scan_result.processing_time_ms,
"Scan job completed"
);
Ok(())
}
Err(e) => {
progress
.report(
JobStatus::Failed(e.to_string()),
"error",
0,
&format!("Gagal: {}", e),
)
.await?;
tracing::error!(job_id = %job.id, error = %e, "Scan job failed");
Err(e)
}
}
}
@@ -0,0 +1,112 @@
use image::GrayImage;
use tools_common::error::PipelineError;
/// OCR result with text and word-level bounding boxes.
pub struct OcrResult {
pub full_text: String,
pub words: Vec<OcrWord>,
pub confidence: f32,
}
/// A single word detected by OCR with its bounding box.
#[derive(Debug, Clone)]
pub struct OcrWord {
pub text: String,
pub bbox: Bbox,
pub confidence: i32,
}
/// Bounding box coordinates.
#[derive(Debug, Clone)]
pub struct Bbox {
pub x: i32,
pub y: i32,
pub width: i32,
pub height: i32,
}
/// Initialize Tesseract OCR engine.
/// Uses leptess crate which binds to libtesseract.
/// Falls back gracefully if Tesseract is not installed.
#[cfg(feature = "tesseract")]
fn init_tesseract(lang: &str) -> Result<leptess::LepTess, PipelineError> {
let tessdata_prefix = std::env::var("TESSDATA_PREFIX")
.unwrap_or_else(|_| "/usr/share/tesseract-ocr/5/tessdata".to_string());
let mut tess = leptess::LepTess::new(Some(&tessdata_prefix), lang)
.map_err(|e| PipelineError::Ocr(format!("Failed to init Tesseract: {}", e)))?;
Ok(tess)
}
/// Run OCR on a grayscale image and return extracted text.
/// Uses Tesseract via leptess crate when the "tesseract" feature is enabled.
/// Falls back to a placeholder when Tesseract is unavailable.
pub fn ocr_text(img: &GrayImage, lang: &str) -> Result<OcrResult, PipelineError> {
#[cfg(feature = "tesseract")]
{
let mut tess = init_tesseract(lang)?;
let width = img.width() as i32;
let height = img.height() as i32;
// Set image from memory
tess.set_image_from_mem(&img.to_vec(), width, height, 1, width)
.map_err(|e| PipelineError::Ocr(format!("Failed to set image: {}", e)))?;
tess.set_source_resolution(300);
// Set PSM to automatic
tess.set_page_seg_mode(3);
let text = tess.get_utf8_text()
.map_err(|e| PipelineError::Ocr(format!("OCR failed: {}", e)))?;
let words = tess.get_words()
.iter()
.map(|w| OcrWord {
text: w.text.clone(),
bbox: Bbox {
x: w.x,
y: w.y,
width: w.w,
height: w.h,
},
confidence: w.confidence,
})
.collect();
let confidence = if words.is_empty() {
0.0
} else {
words.iter().map(|w| w.confidence as f32).sum::<f32>() / words.len() as f32
};
Ok(OcrResult {
full_text: text,
words,
confidence,
})
}
#[cfg(not(feature = "tesseract"))]
{
tracing::warn!("Tesseract feature not enabled, OCR returning placeholder");
Ok(OcrResult {
full_text: String::new(),
words: Vec::new(),
confidence: 0.0,
})
}
}
/// Run OCR on a grayscale image, returning only the text.
pub fn ocr_text_only(img: &GrayImage, lang: &str) -> Result<String, PipelineError> {
ocr_text(img, lang).map(|r| r.full_text)
}
/// Run OCR with word-level bounding boxes.
pub fn ocr_words(img: &GrayImage, lang: &str) -> Result<Vec<OcrWord>, PipelineError> {
ocr_text(img, lang).map(|r| r.words)
}
@@ -0,0 +1,190 @@
use image::GrayImage;
use lopdf::{Document, Object, Stream, Dictionary};
use tools_common::error::PipelineError;
/// A4 page dimensions in points (1 pt = 1/72 inch).
pub const A4_WIDTH_PT: f64 = 595.28;
pub const A4_HEIGHT_PT: f64 = 841.89;
/// Generate a searchable PDF with JPEG image + invisible OCR text layer.
pub fn generate_searchable_pdf(
image_data: &[u8],
_ocr_text: &str,
words: &[super::ocr::OcrWord],
page_width: f64,
page_height: f64,
) -> Result<Vec<u8>, PipelineError> {
let mut doc = Document::new();
// ── Pages object ──
let pages_id = doc.new_object_id();
let mut pages = Dictionary::new();
pages.set("Type", Object::Name("Pages".as_bytes().to_vec()));
pages.set("Kids", Object::Array(vec![]));
pages.set("Count", Object::Integer(0));
doc.objects.insert(pages_id, Object::Dictionary(pages));
// ── Image XObject ──
let mut img_dict = Dictionary::new();
img_dict.set("Type", Object::Name("XObject".as_bytes().to_vec()));
img_dict.set("Subtype", Object::Name("Image".as_bytes().to_vec()));
img_dict.set("Width", Object::Integer(page_width as i64));
img_dict.set("Height", Object::Integer(page_height as i64));
img_dict.set("ColorSpace", Object::Name("DeviceGray".as_bytes().to_vec()));
img_dict.set("BitsPerComponent", Object::Integer(8));
img_dict.set("Filter", Object::Name("DCTDecode".as_bytes().to_vec()));
let image_stream = Stream::new(img_dict, image_data.to_vec());
let image_id = doc.add_object(Object::Stream(image_stream));
// ── Content stream: place image + invisible text ──
let mut content = Vec::new();
// Place image at full page
content.extend_from_slice(b"q\n");
content.extend_from_slice(
format!("{} 0 0 {} 0 0 cm\n", page_width, page_height).as_bytes(),
);
content.extend_from_slice(b"/Im0 Do\n");
content.extend_from_slice(b"Q\n");
// Add invisible text layer (searchable)
for word in words {
let x = word.bbox.x as f64 / 300.0 * 72.0;
let y = page_height - (word.bbox.y as f64 / 300.0 * 72.0);
let font_size = (word.bbox.height as f64 / 300.0 * 72.0 * 0.8).max(4.0);
content.extend_from_slice(b"BT\n");
content.extend_from_slice(b"3 Tr\n"); // Rendering mode: invisible (neither fill nor stroke)
content.extend_from_slice(
format!("/F1 {} Tf\n{} {} Td\n", font_size, x, y - font_size).as_bytes(),
);
content.extend_from_slice(
format!("({}) Tj\n", escape_pdf_string(&word.text)).as_bytes(),
);
content.extend_from_slice(b"ET\n");
}
let content_stream = Stream::new(Dictionary::new(), content);
let content_id = doc.add_object(Object::Stream(content_stream));
// ── Font dictionary ──
let mut font_dict = Dictionary::new();
let mut f1 = Dictionary::new();
f1.set("Type", Object::Name("Font".as_bytes().to_vec()));
f1.set("Subtype", Object::Name("Type1".as_bytes().to_vec()));
f1.set("BaseFont", Object::Name("Helvetica".as_bytes().to_vec()));
font_dict.set("F1", Object::Dictionary(f1));
// ── Resources dictionary ──
let mut xobject_dict = Dictionary::new();
xobject_dict.set("Im0", Object::Reference(image_id));
let mut resources = Dictionary::new();
resources.set("XObject", Object::Dictionary(xobject_dict));
resources.set("Font", Object::Dictionary(font_dict));
// ── Page object ──
let page_id = doc.new_object_id();
let mut page = Dictionary::new();
page.set("Type", Object::Name("Page".as_bytes().to_vec()));
page.set("Parent", Object::Reference(pages_id));
page.set(
"MediaBox",
Object::Array(vec![
Object::Real(0.0),
Object::Real(0.0),
Object::Real(page_width as f32),
Object::Real(page_height as f32),
]),
);
page.set("Contents", Object::Reference(content_id));
page.set("Resources", Object::Dictionary(resources));
doc.objects.insert(page_id, Object::Dictionary(page));
// ── Update pages object ──
if let Some(Object::Dictionary(ref mut pages_dict)) = doc.objects.get_mut(&pages_id) {
pages_dict.set("Count", Object::Integer(1));
pages_dict.set("Kids", Object::Array(vec![Object::Reference(page_id)]));
}
// ── Save ──
let mut output = Vec::new();
doc.save_to(&mut output)
.map_err(|e| PipelineError::PdfGeneration(e.to_string()))?;
Ok(output)
}
/// Escape special characters for PDF string literals.
fn escape_pdf_string(s: &str) -> String {
let mut result = String::with_capacity(s.len());
for c in s.chars() {
match c {
'(' => result.push_str("\\("),
')' => result.push_str("\\)"),
'\\' => result.push_str("\\\\"),
'\n' => result.push_str("\\n"),
'\r' => result.push_str("\\r"),
'\t' => result.push_str("\\t"),
other => result.push(other),
}
}
result
}
/// Compress grayscale image as JPEG bytes.
pub fn compress_image_jpeg(img: &GrayImage, quality: u8) -> Result<Vec<u8>, PipelineError> {
let mut bytes = Vec::new();
let rgb = image::DynamicImage::ImageLuma8(img.clone()).into_rgb8();
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, quality);
encoder
.encode(
rgb.as_raw(),
img.width(),
img.height(),
image::ExtendedColorType::Rgb8,
)
.map_err(|e| PipelineError::PdfGeneration(format!("JPEG compression failed: {}", e)))?;
Ok(bytes)
}
/// Compress RGB image data as JPEG bytes.
pub fn compress_rgb_image_jpeg(
data: &[u8],
width: u32,
height: u32,
quality: u8,
) -> Result<Vec<u8>, PipelineError> {
let mut bytes = Vec::new();
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut bytes, quality);
encoder
.encode(data, width, height, image::ExtendedColorType::Rgb8)
.map_err(|e| PipelineError::PdfGeneration(format!("JPEG compression failed: {}", e)))?;
Ok(bytes)
}
#[cfg(test)]
mod tests {
use super::*;
use image::Luma;
#[test]
fn test_escape_pdf_string() {
assert_eq!(escape_pdf_string("hello"), "hello");
assert_eq!(escape_pdf_string("(parens)"), "\\(parens\\)");
assert_eq!(escape_pdf_string("back\\slash"), "back\\\\slash");
}
#[test]
fn test_jpeg_compression() {
let img = GrayImage::from_pixel(100, 100, Luma([128]));
let result = compress_image_jpeg(&img, 90);
assert!(result.is_ok(), "JPEG compression failed: {:?}", result.err());
let bytes = result.unwrap();
assert!(!bytes.is_empty());
assert_eq!(&bytes[0..2], &[0xFF, 0xD8]);
}
}
@@ -0,0 +1,116 @@
use std::path::Path;
use std::time::Instant;
use image::DynamicImage;
use tools_common::error::PipelineError;
use tools_common::types::Job;
use crate::config::WorkerConfig;
use crate::nats::progress::ProgressReporter;
use super::binarize::sauvola_threshold;
use super::corners::detect_corners_with_fallback;
use super::deskew::deskew;
use super::edge::detect_edges;
use super::enhance::enhance_final;
use super::preprocess::preprocess;
use super::shadow::remove_shadow;
use super::warp::warp_perspective;
/// Result of the scanning pipeline.
pub struct ScanResult {
pub output_path: String,
pub page_count: u32,
pub file_size: u64,
pub ocr_text: Option<String>,
pub processing_time_ms: u64,
}
/// Run the full scanner pipeline with all stages.
pub async fn process(
job: &Job,
config: &WorkerConfig,
progress: &ProgressReporter,
) -> Result<ScanResult, Box<dyn std::error::Error + Send + Sync>> {
let start = Instant::now();
let input_path = Path::new(&job.file_path);
// Create output directory
let output_dir = config.storage_path.join("output");
tokio::fs::create_dir_all(&output_dir).await?;
// Stage 1: Load & Preprocess (0-15%)
report(progress, "preprocess", 5, "Memuat dan meresize gambar...").await;
let gray = preprocess(input_path)
.map_err(|e| format!("Preprocess failed: {}", e))?;
// Stage 2: Edge Detection (15-30%)
report(progress, "edge_detection", 20, "Mendeteksi tepi dokumen...").await;
let edges = detect_edges(&gray).map_err(|e| format!("Edge detection failed: {}", e))?;
// Stage 3: Corner Detection (30-40%)
report(progress, "corner_detection", 35, "Mencari sudut dokumen...").await;
let corners = detect_corners_with_fallback(&edges)?;
// Stage 4: Perspective Warp (40-55%)
report(progress, "warp", 45, "Meluruskan perspektif dokumen...").await;
let image = image::open(input_path)
.map_err(|e| PipelineError::ImageLoad(e.to_string()))?;
let warped = warp_perspective(&image, corners)?;
// Stage 5: Shadow Removal (55-70%)
report(progress, "shadow_removal", 60, "Menghilangkan bayangan...").await;
let warped_gray = warped.to_luma8();
let clean = remove_shadow(&warped_gray);
// Stage 6: Binarization (70-80%)
report(progress, "binarization", 75, "Mengubah ke hitam-putih...").await;
let binary = sauvola_threshold(&clean, 30, 0.2);
// Stage 7: Deskew (80-87%)
report(progress, "deskew", 82, "Meluruskan teks...").await;
let final_img = deskew(&binary);
// Stage 8: Enhance (87-93%)
report(progress, "enhance", 90, "Mengoptimalkan kualitas...").await;
let final_img = enhance_final(&final_img);
// Stage 9: Save output (93-100%)
report(progress, "save", 95, "Menyimpan hasil...").await;
let output_filename = format!("{}.png", progress.job_id());
let output_path = output_dir.join(&output_filename);
final_img.save(&output_path)?;
let elapsed = start.elapsed().as_millis() as u64;
tracing::info!(
job_id = %progress.job_id(),
duration_ms = elapsed,
"Pipeline complete"
);
Ok(ScanResult {
output_path: output_path.to_string_lossy().to_string(),
page_count: 1,
file_size: tokio::fs::metadata(&output_path).await.map(|m| m.len()).unwrap_or(0),
ocr_text: None,
processing_time_ms: elapsed,
})
}
/// Helper to report progress.
async fn report(progress: &ProgressReporter, stage: &str, pct: u8, msg: &str) {
let _ = progress
.report(
tools_common::types::JobStatus::Processing {
stage: stage.to_string(),
progress: pct,
},
stage,
pct,
msg,
)
.await;
}
@@ -0,0 +1,74 @@
use image::{DynamicImage, GrayImage, Luma};
use image::imageops::FilterType;
use tools_common::error::PipelineError;
/// Maximum dimension for processing (edge detection works fine at this resolution).
const MAX_DIMENSION: u32 = 2000;
/// Load image from file path.
pub fn load_image(path: &std::path::Path) -> Result<DynamicImage, PipelineError> {
image::open(path).map_err(|e| PipelineError::ImageLoad(e.to_string()))
}
/// Resize image if it exceeds the maximum dimension, preserving aspect ratio.
/// Uses Lanczos3 filter for sharpest downscale.
pub fn safe_resize(img: &DynamicImage) -> DynamicImage {
let (w, h) = (img.width(), img.height());
let max_dim = w.max(h) as f64;
if max_dim > MAX_DIMENSION as f64 {
let scale = MAX_DIMENSION as f64 / max_dim;
let new_w = (w as f64 * scale) as u32;
let new_h = (h as f64 * scale) as u32;
img.resize_exact(new_w.max(1), new_h.max(1), FilterType::Lanczos3)
} else {
img.clone()
}
}
/// Convert to grayscale (Luma8).
pub fn to_grayscale(img: &DynamicImage) -> GrayImage {
img.to_luma8()
}
/// Full preprocess pipeline: load → resize → grayscale.
pub fn preprocess(path: &std::path::Path) -> Result<GrayImage, PipelineError> {
let img = load_image(path)?;
let resized = safe_resize(&img);
Ok(to_grayscale(&resized))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_safe_resize_no_resize() {
// Image smaller than MAX_DIMENSION should not be resized
let img = DynamicImage::new_luma8(800, 600);
let result = safe_resize(&img);
assert_eq!(result.width(), 800);
assert_eq!(result.height(), 600);
}
#[test]
fn test_safe_resize_downscale() {
// 12MP image (4000x3000) should be resized to ≤2000px
let img = DynamicImage::new_luma8(4000, 3000);
let result = safe_resize(&img);
assert!(result.width() <= 2000);
assert!(result.height() <= 2000);
// Aspect ratio preserved: 4000/3000 = 1.333
let ratio = result.width() as f64 / result.height() as f64;
assert!((ratio - 4.0 / 3.0).abs() < 0.01);
}
#[test]
fn test_to_grayscale() {
let img = DynamicImage::new_rgba8(100, 100);
let gray = to_grayscale(&img);
assert_eq!(gray.width(), 100);
assert_eq!(gray.height(), 100);
}
}
@@ -0,0 +1,161 @@
use image::{GrayImage, Luma};
use imageproc::filter::gaussian_blur_f32;
/// Remove uneven lighting and shadows from a grayscale document image.
///
/// Algorithm:
/// 1. Large Gaussian blur to estimate background illumination
/// 2. Subtract background from original
/// 3. Apply CLAHE for local contrast normalization
pub fn remove_shadow(img: &GrayImage) -> GrayImage {
let (w, h) = (img.width(), img.height());
// 1. Large Gaussian blur for illumination estimate
let blur_radius = (w.min(h) as f64 / 50.0).max(15.0);
let background = gaussian_blur_f32(img, blur_radius as f32);
// 2. Subtract background
let bg_mean = mean_pixel(&background);
let mut corrected = GrayImage::new(w, h);
for y in 0..h {
for x in 0..w {
let orig = img.get_pixel(x, y)[0] as f32;
let bg = background.get_pixel(x, y)[0] as f32;
let corrected_val = (orig - bg + bg_mean).clamp(0.0, 255.0) as u8;
corrected.put_pixel(x, y, Luma([corrected_val]));
}
}
// 3. Apply CLAHE
apply_clahe(&corrected, 8, 4)
}
/// Compute mean pixel value of a grayscale image.
fn mean_pixel(img: &GrayImage) -> f32 {
let sum: u32 = img.iter().map(|&p| p as u32).sum();
let count = img.width() * img.height();
if count > 0 {
sum as f32 / count as f32
} else {
0.0
}
}
/// Contrast Limited Adaptive Histogram Equalization.
/// Divides the image into tiles and applies histogram equalization to each.
fn apply_clahe(img: &GrayImage, tile_size: u32, clip_limit: u8) -> GrayImage {
let (w, h) = (img.width(), img.height());
let tiles_x = (w + tile_size - 1) / tile_size;
let tiles_y = (h + tile_size - 1) / tile_size;
let mut output = GrayImage::new(w, h);
for ty in 0..tiles_y {
for tx in 0..tiles_x {
let start_x = tx * tile_size;
let start_y = ty * tile_size;
let end_x = (start_x + tile_size).min(w);
let end_y = (start_y + tile_size).min(h);
// Compute histogram for this tile
let mut hist = [0u32; 256];
for y in start_y..end_y {
for x in start_x..end_x {
hist[img.get_pixel(x, y)[0] as usize] += 1;
}
}
// Clip histogram
let tile_pixels = (end_x - start_x) * (end_y - start_y);
let clip_limit_count = tile_pixels as u32 * clip_limit as u32 / 255 / 10;
let mut excess = 0u32;
for count in hist.iter_mut() {
if *count > clip_limit_count {
excess += *count - clip_limit_count;
*count = clip_limit_count;
}
}
// Redistribute excess
let add_per_bin = excess / 256;
for count in hist.iter_mut() {
*count += add_per_bin;
}
// Build CDF
let mut cdf = [0u32; 256];
cdf[0] = hist[0];
for i in 1..256 {
cdf[i] = cdf[i - 1] + hist[i];
}
let cdf_min = cdf.iter().find(|&&v| v > 0).copied().unwrap_or(0);
// Apply equalization to this tile
for y in start_y..end_y {
for x in start_x..end_x {
let pixel = img.get_pixel(x, y)[0] as usize;
let equalized = if cdf_max(cdf) > cdf_min {
((cdf[pixel].saturating_sub(cdf_min)) as f64
/ (cdf_max(cdf).saturating_sub(cdf_min)) as f64
* 255.0) as u8
} else {
pixel as u8
};
output.put_pixel(x, y, Luma([equalized]));
}
}
}
}
output
}
/// Get the maximum value in the CDF array.
fn cdf_max(cdf: [u32; 256]) -> u32 {
*cdf.iter().max().unwrap_or(&0)
}
/// Retinex-based shadow removal (alternative algorithm).
#[allow(dead_code)]
fn retinex_shadow_removal(img: &GrayImage) -> GrayImage {
let (w, h) = (img.width(), img.height());
let blurred = gaussian_blur_f32(img, 30.0);
let mut output = GrayImage::new(w, h);
for y in 0..h {
for x in 0..w {
let orig = img.get_pixel(x, y)[0] as f32;
let bg = blurred.get_pixel(x, y)[0] as f32;
if bg > 0.0 {
let retinex = (orig / bg).ln() * 255.0;
output.put_pixel(x, y, Luma([retinex.clamp(0.0, 255.0) as u8]));
} else {
output.put_pixel(x, y, Luma([0]));
}
}
}
output
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_shadow_removal_uniform() {
// Uniform image should remain uniform
let img = GrayImage::from_pixel(100, 100, Luma([128]));
let result = remove_shadow(&img);
assert_eq!(result.width(), 100);
assert_eq!(result.height(), 100);
// The result should have fewer dark pixels than a shadowed version
let dark_count = result.iter().filter(|&&p| p < 50).count();
assert!(dark_count < 100); // Very few dark pixels
}
#[test]
fn test_mean_pixel() {
let img = GrayImage::from_pixel(10, 10, Luma([100]));
assert!((mean_pixel(&img) - 100.0).abs() < 1.0);
}
}
@@ -0,0 +1,208 @@
use image::{DynamicImage, GrayImage, Luma};
use nalgebra::{Matrix3, SVD};
use tools_common::error::PipelineError;
use crate::scanner::corners::CornerPoint;
/// Compute homography matrix from 4 point correspondences using DLT algorithm.
pub fn compute_homography(
src: &[CornerPoint; 4],
dst: &[CornerPoint; 4],
) -> Result<[[f64; 3]; 3], PipelineError> {
// Build 8x9 matrix A from 4 point correspondences
// Each correspondence (x,y) -> (x',y') gives 2 rows:
// [-x, -y, -1, 0, 0, 0, x*x', y*x', x']
// [ 0, 0, 0, -x, -y, -1, x*y', y*y', y']
let mut a = nalgebra::DMatrix::<f64>::zeros(8, 9);
for i in 0..4 {
let x = src[i].0;
let y = src[i].1;
let xp = dst[i].0;
let yp = dst[i].1;
// First row
a[(i * 2, 0)] = -x;
a[(i * 2, 1)] = -y;
a[(i * 2, 2)] = -1.0;
a[(i * 2, 3)] = 0.0;
a[(i * 2, 4)] = 0.0;
a[(i * 2, 5)] = 0.0;
a[(i * 2, 6)] = x * xp;
a[(i * 2, 7)] = y * xp;
a[(i * 2, 8)] = xp;
// Second row
a[(i * 2 + 1, 0)] = 0.0;
a[(i * 2 + 1, 1)] = 0.0;
a[(i * 2 + 1, 2)] = 0.0;
a[(i * 2 + 1, 3)] = -x;
a[(i * 2 + 1, 4)] = -y;
a[(i * 2 + 1, 5)] = -1.0;
a[(i * 2 + 1, 6)] = x * yp;
a[(i * 2 + 1, 7)] = y * yp;
a[(i * 2 + 1, 8)] = yp;
}
// Solve Ah = 0 via SVD: h = last column of V
let svd = SVD::new(a, true, true);
if let Some(v_t) = &svd.v_t {
let nrows = v_t.nrows();
if nrows > 0 {
let h_vec: Vec<f64> = v_t.row(nrows - 1).iter().copied().collect();
if h_vec.len() >= 9 {
let h = [
[h_vec[0], h_vec[1], h_vec[2]],
[h_vec[3], h_vec[4], h_vec[5]],
[h_vec[6], h_vec[7], h_vec[8]],
];
return Ok(h);
}
}
}
Err(PipelineError::Warp("SVD decomposition failed".to_string()))
}
/// Invert a 3x3 homography matrix.
pub fn invert_homography(h: &[[f64; 3]; 3]) -> [[f64; 3]; 3] {
let m = Matrix3::new(h[0][0], h[0][1], h[0][2], h[1][0], h[1][1], h[1][2], h[2][0], h[2][1], h[2][2]);
let inv = m
.try_inverse()
.unwrap_or(Matrix3::identity());
[
[inv[(0, 0)], inv[(0, 1)], inv[(0, 2)]],
[inv[(1, 0)], inv[(1, 1)], inv[(1, 2)]],
[inv[(2, 0)], inv[(2, 1)], inv[(2, 2)]],
]
}
/// Apply homography to a point (forward mapping).
pub fn apply_homography(h: &[[f64; 3]; 3], x: f64, y: f64) -> (f64, f64) {
let z = h[2][0] * x + h[2][1] * y + h[2][2];
if z.abs() < 1e-10 {
return (x, y);
}
let xp = (h[0][0] * x + h[0][1] * y + h[0][2]) / z;
let yp = (h[1][0] * x + h[1][1] * y + h[1][2]) / z;
(xp, yp)
}
/// Bilinear interpolation at sub-pixel coordinates.
fn bilinear_interpolate(img: &GrayImage, x: f64, y: f64) -> Luma<u8> {
let x0 = x.floor() as i32;
let y0 = y.floor() as i32;
let x1 = x0 + 1;
let y1 = y0 + 1;
let w = img.width() as i32;
let h = img.height() as i32;
// Clamp coordinates
let x0 = x0.clamp(0, w - 1);
let x1 = x1.clamp(0, w - 1);
let y0 = y0.clamp(0, h - 1);
let y1 = y1.clamp(0, h - 1);
let fx = x - x0 as f64;
let fy = y - y0 as f64;
let p00 = img.get_pixel(x0 as u32, y0 as u32)[0] as f64;
let p10 = img.get_pixel(x1 as u32, y0 as u32)[0] as f64;
let p01 = img.get_pixel(x0 as u32, y1 as u32)[0] as f64;
let p11 = img.get_pixel(x1 as u32, y1 as u32)[0] as f64;
let val = p00 * (1.0 - fx) * (1.0 - fy)
+ p10 * fx * (1.0 - fy)
+ p01 * (1.0 - fx) * fy
+ p11 * fx * fy;
Luma([val.round().clamp(0.0, 255.0) as u8])
}
/// Apply perspective warp to correct the document perspective.
/// Takes the original color image and 4 corners, returns warped image.
pub fn warp_perspective(
img: &DynamicImage,
corners: [CornerPoint; 4],
) -> Result<DynamicImage, PipelineError> {
let [tl, tr, br, bl] = corners;
// Compute target width and height (preserve aspect ratio)
let width_top = distance(tl, tr);
let width_bot = distance(bl, br);
let width = width_top.max(width_bot).ceil() as u32;
let height_left = distance(tl, bl);
let height_right = distance(tr, br);
let height = height_left.max(height_right).ceil() as u32;
// Clamp output dimensions
let width = width.min(3000).max(1);
let height = height.min(3000).max(1);
let src = [tl, tr, br, bl];
let dst = [
(0.0, 0.0),
(width as f64, 0.0),
(width as f64, height as f64),
(0.0, height as f64),
];
let h = compute_homography(&src, &dst)?;
let h_inv = invert_homography(&h);
let gray = img.to_luma8();
let mut output = GrayImage::new(width, height);
// Backward mapping: for each output pixel, find source pixel
for y in 0..height {
for x in 0..width {
let (sx, sy) = apply_homography(&h_inv, x as f64, y as f64);
let pixel = bilinear_interpolate(&gray, sx, sy);
output.put_pixel(x, y, pixel);
}
}
Ok(DynamicImage::ImageLuma8(output))
}
/// Euclidean distance between two points.
fn distance(a: CornerPoint, b: CornerPoint) -> f64 {
((a.0 - b.0).powi(2) + (a.1 - b.1).powi(2)).sqrt()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_homography_identity() {
// Identity mapping should produce identity matrix
let src = [(0.0, 0.0), (100.0, 0.0), (100.0, 100.0), (0.0, 100.0)];
let dst = [(0.0, 0.0), (100.0, 0.0), (100.0, 100.0), (0.0, 100.0)];
let h = compute_homography(&src, &dst).unwrap();
let (xp, yp) = apply_homography(&h, 50.0, 50.0);
assert!((xp - 50.0).abs() < 1.0);
assert!((yp - 50.0).abs() < 1.0);
}
#[test]
fn test_invert_homography() {
let h = [[2.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 0.0, 1.0]];
let inv = invert_homography(&h);
let (xp, yp) = apply_homography(&inv, 100.0, 100.0);
assert!((xp - 50.0).abs() < 0.001);
assert!((yp - 50.0).abs() < 0.001);
}
#[test]
fn test_bilinear_interpolate() {
let mut img = GrayImage::new(3, 3);
img.put_pixel(0, 0, Luma([100]));
img.put_pixel(1, 0, Luma([200]));
let pixel = bilinear_interpolate(&img, 0.5, 0.0);
assert_eq!(pixel[0], 150); // Midpoint between 100 and 200
}
}
@@ -0,0 +1,81 @@
use redis::AsyncCommands;
/// Cleanup expired files and Redis keys.
/// Scans storage directory and removes files older than TTL.
pub struct CleanupScheduler;
impl CleanupScheduler {
/// Run a single cleanup cycle.
pub async fn run(
storage_path: &std::path::Path,
redis_client: &redis::Client,
ttl_seconds: u64,
) -> Result<CleanupResult, Box<dyn std::error::Error + Send + Sync>> {
let mut result = CleanupResult::default();
let now = std::time::SystemTime::now();
// Clean up upload files
let upload_dir = storage_path.join("upload");
if upload_dir.exists() {
let mut entries = tokio::fs::read_dir(&upload_dir).await?;
while let Some(entry) = entries.next_entry().await? {
if let Ok(metadata) = entry.metadata().await {
if let Ok(modified) = metadata.modified() {
if now
.duration_since(modified)
.map(|d| d.as_secs() > ttl_seconds)
.unwrap_or(false)
{
if let Ok(_) = tokio::fs::remove_file(entry.path()).await {
result.files_deleted += 1;
result.bytes_freed += metadata.len();
}
}
}
}
}
}
// Clean up output files
let output_dir = storage_path.join("output");
if output_dir.exists() {
let mut entries = tokio::fs::read_dir(&output_dir).await?;
while let Some(entry) = entries.next_entry().await? {
if let Ok(metadata) = entry.metadata().await {
if let Ok(modified) = metadata.modified() {
if now
.duration_since(modified)
.map(|d| d.as_secs() > ttl_seconds)
.unwrap_or(false)
{
if let Ok(_) = tokio::fs::remove_file(entry.path()).await {
result.files_deleted += 1;
result.bytes_freed += metadata.len();
}
}
}
}
}
}
// Clean up orphaned Redis keys
if let Ok(mut conn) = redis_client.get_multiplexed_async_connection().await {
// Scan for expired job keys
let _: Result<(), _> = redis::cmd("SCAN")
.arg(0)
.arg("MATCH")
.arg("job:*")
.query_async(&mut conn)
.await;
}
Ok(result)
}
}
#[derive(Debug, Default)]
pub struct CleanupResult {
pub files_deleted: u64,
pub bytes_freed: u64,
pub orphan_keys: u64,
}
@@ -0,0 +1,3 @@
/// Auto-cleanup scheduler for expired files and Redis keys.
/// TODO: Phase 1.4 - implement cleanup logic
pub mod cleanup;
@@ -0,0 +1,2 @@
// Video processing module.
// TODO: Phase 4 - implement compress, extract audio, trim, GIF maker