feat(infra): add tools service with document scanner, image & PDF tools

Implement self-hosted document scanner and media processing tools as an alternative to CamScanner/ilovepdf without third-party uploads. Backend: Rust Axum gateway + worker pool with NATS JetStream queue Frontend: Next.js 16 + shadcn/ui + Tailwind v4 + Framer Motion Pipeline: Canny edge detection -> DLT homography warp -> Sauvola binarization -> Hough deskew -> Tesseract OCR -> searchable PDF Phase 1 (MVP) delivers: - Document scanner with perspective correction and OCR - Image compress/resize/convert tools - PDF merge/split/compress tools - Real-time WebSocket progress updates - Rate limiting, auto-cleanup, Prometheus metrics - Full CI/CD pipeline with Docker multi-stage build Co-Authored-By: Kilo <kilo@kilo.ai>
2026-07-24 13:08:09 +07:00
parent 9c12d135e4
commit 67288c8723
102 changed files with 11527 additions and 3 deletions
@@ -0,0 +1,15 @@
+[package]
+name = "tools-common"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+serde.workspace = true
+serde_json.workspace = true
+uuid.workspace = true
+chrono.workspace = true
+thiserror.workspace = true
+tracing.workspace = true
+async-nats.workspace = true
+redis.workspace = true
@@ -0,0 +1,104 @@
+use thiserror::Error;
+
+/// Errors that can occur during file upload.
+#[derive(Debug, Error)]
+pub enum UploadError {
+    #[error("Invalid MIME type: {0}")]
+    InvalidMime(String),
+
+    #[error("File too large: {0} bytes exceeds maximum of {1} bytes")]
+    FileTooLarge(u64, u64),
+
+    #[error("I/O error: {0}")]
+    Io(#[from] std::io::Error),
+
+    #[error("Virus or suspicious content detected")]
+    VirusDetected,
+
+    #[error("Invalid tool: {0}")]
+    InvalidTool(String),
+
+    #[error("Missing file in upload")]
+    MissingFile,
+
+    #[error("Missing tool parameter")]
+    MissingTool,
+
+    #[error("Serialization error: {0}")]
+    Serde(#[from] serde_json::Error),
+}
+
+/// Errors during processing pipeline execution.
+#[derive(Debug, Error)]
+pub enum PipelineError {
+    #[error("Failed to load image: {0}")]
+    ImageLoad(String),
+
+    #[error("Edge detection failed: {0}")]
+    EdgeDetection(String),
+
+    #[error("Corner detection failed: {0}")]
+    CornerDetection(String),
+
+    #[error("Perspective warp failed: {0}")]
+    Warp(String),
+
+    #[error("Shadow removal failed: {0}")]
+    ShadowRemoval(String),
+
+    #[error("Binarization failed: {0}")]
+    Binarization(String),
+
+    #[error("OCR processing failed: {0}")]
+    Ocr(String),
+
+    #[error("PDF generation failed: {0}")]
+    PdfGeneration(String),
+
+    #[error("Pipeline timed out")]
+    Timeout,
+
+    #[error("Internal error: {0}")]
+    Internal(String),
+}
+
+/// Errors related to NATS messaging.
+#[derive(Debug, Error)]
+pub enum NatsError {
+    #[error("Failed to publish message: {0}")]
+    Publish(String),
+
+    #[error("Failed to subscribe: {0}")]
+    Subscribe(String),
+
+    #[error("JetStream error: {0}")]
+    JetStream(String),
+
+    #[error("Connection timeout")]
+    Timeout,
+
+    #[error("NATS connection error: {0}")]
+    Connection(String),
+}
+
+/// Errors related to Redis operations.
+#[derive(Debug, Error)]
+pub enum RedisError {
+    #[error("Redis connection failed: {0}")]
+    Connection(String),
+
+    #[error("Redis query failed: {0}")]
+    Query(String),
+
+    #[error("Serialization error: {0}")]
+    Serialization(#[from] serde_json::Error),
+
+    #[error("Key not found: {0}")]
+    NotFound(String),
+}
+
+impl From<redis::RedisError> for RedisError {
+    fn from(e: redis::RedisError) -> Self {
+        RedisError::Query(e.to_string())
+    }
+}
@@ -0,0 +1,3 @@
+pub mod error;
+pub mod nats;
+pub mod types;
@@ -0,0 +1,104 @@
+/// NATS subject constants for the tools service.
+///
+/// Subject naming convention:
+///   tools.<tool_group>.jobs.{job_id}      — Job submission queue
+///   tools.<tool_group>.progress.{job_id}  — Progress update fan-out
+///   tools.scheduler.cleanup               — Cron-triggered cleanup
+
+// ── Job Subjects ──
+
+pub const SCAN_JOBS: &str = "tools.scan.jobs";
+pub const SCAN_PROGRESS: &str = "tools.scan.progress";
+pub const IMAGE_JOBS: &str = "tools.image.jobs";
+pub const IMAGE_PROGRESS: &str = "tools.image.progress";
+pub const PDF_JOBS: &str = "tools.pdf.jobs";
+pub const PDF_PROGRESS: &str = "tools.pdf.progress";
+pub const VIDEO_JOBS: &str = "tools.video.jobs";
+pub const VIDEO_PROGRESS: &str = "tools.video.progress";
+pub const AUDIO_JOBS: &str = "tools.audio.jobs";
+pub const AUDIO_PROGRESS: &str = "tools.audio.progress";
+
+// ── Scheduler Subjects ──
+
+pub const SCHEDULER_CLEANUP: &str = "tools.scheduler.cleanup";
+
+// ── Stream Names ──
+
+pub const STREAM_JOBS: &str = "tools-jobs";
+pub const STREAM_PROGRESS: &str = "tools-progress";
+
+// ── Stream Configuration ──
+
+/// Returns the stream configuration for jobs.
+/// Max age: 24h, storage: file (persistent on disk).
+pub fn jobs_stream_config() -> async_nats::jetstream::stream::Config {
+    use async_nats::jetstream::stream::Config;
+    Config {
+        name: STREAM_JOBS.to_string(),
+        subjects: vec![
+            "tools.scan.jobs.*".to_string(),
+            "tools.image.jobs.*".to_string(),
+            "tools.pdf.jobs.*".to_string(),
+            "tools.video.jobs.*".to_string(),
+            "tools.audio.jobs.*".to_string(),
+            "tools.scheduler.>".to_string(),
+        ],
+        max_age: std::time::Duration::from_secs(24 * 3600),
+        storage: async_nats::jetstream::stream::StorageType::File,
+        ..Default::default()
+    }
+}
+
+/// Returns the stream configuration for progress events.
+/// Max age: 1h, storage: memory (no persistence needed).
+pub fn progress_stream_config() -> async_nats::jetstream::stream::Config {
+    use async_nats::jetstream::stream::Config;
+    Config {
+        name: STREAM_PROGRESS.to_string(),
+        subjects: vec![
+            "tools.scan.progress.*".to_string(),
+            "tools.image.progress.*".to_string(),
+            "tools.pdf.progress.*".to_string(),
+            "tools.video.progress.*".to_string(),
+            "tools.audio.progress.*".to_string(),
+        ],
+        max_age: std::time::Duration::from_secs(3600),
+        storage: async_nats::jetstream::stream::StorageType::Memory,
+        ..Default::default()
+    }
+}
+
+/// Build a job subject for a given tool and job ID.
+pub fn job_subject(tool_group: &str, job_id: &str) -> String {
+    format!("tools.{}.jobs.{}", tool_group, job_id)
+}
+
+/// Build a progress subject for a given tool and job ID.
+pub fn progress_subject(tool_group: &str, job_id: &str) -> String {
+    format!("tools.{}.progress.{}", tool_group, job_id)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_subject_format() {
+        assert_eq!(job_subject("scan", "abc-123"), "tools.scan.jobs.abc-123");
+        assert_eq!(
+            progress_subject("scan", "abc-123"),
+            "tools.scan.progress.abc-123"
+        );
+        assert_eq!(
+            job_subject("image", "def-456"),
+            "tools.image.jobs.def-456"
+        );
+        assert_eq!(SCHEDULER_CLEANUP, "tools.scheduler.cleanup");
+    }
+
+    #[test]
+    fn test_stream_names() {
+        assert_eq!(STREAM_JOBS, "tools-jobs");
+        assert_eq!(STREAM_PROGRESS, "tools-progress");
+    }
+}
@@ -0,0 +1,228 @@
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+/// Status of a processing job.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub enum JobStatus {
+    Queued,
+    Processing {
+        stage: String,
+        progress: u8,
+    },
+    Completed,
+    NeedsManualCrop,
+    Failed(String),
+}
+
+/// Available tool types.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub enum Tool {
+    Scan,
+    ImageCompress,
+    ImageResize,
+    ImageConvert,
+    RemoveBg,
+    PdfMerge,
+    PdfSplit,
+    ImagesToPdf,
+    PdfCompress,
+    PdfToImages,
+    VideoCompress,
+    AudioExtract,
+    VideoTrim,
+    GifMaker,
+    AudioConvert,
+}
+
+impl Tool {
+    /// Returns the NATS subject prefix for this tool.
+    pub fn subject_prefix(&self) -> &'static str {
+        match self {
+            Tool::Scan => "tools.scan",
+            Tool::ImageCompress
+            | Tool::ImageResize
+            | Tool::ImageConvert
+            | Tool::RemoveBg => "tools.image",
+            Tool::PdfMerge
+            | Tool::PdfSplit
+            | Tool::ImagesToPdf
+            | Tool::PdfCompress
+            | Tool::PdfToImages => "tools.pdf",
+            Tool::VideoCompress | Tool::VideoTrim | Tool::GifMaker => "tools.video",
+            Tool::AudioExtract | Tool::AudioConvert => "tools.audio",
+        }
+    }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Tool::Scan => "scan",
+            Tool::ImageCompress => "image-compress",
+            Tool::ImageResize => "image-resize",
+            Tool::ImageConvert => "image-convert",
+            Tool::RemoveBg => "remove-bg",
+            Tool::PdfMerge => "pdf-merge",
+            Tool::PdfSplit => "pdf-split",
+            Tool::ImagesToPdf => "images-to-pdf",
+            Tool::PdfCompress => "pdf-compress",
+            Tool::PdfToImages => "pdf-to-images",
+            Tool::VideoCompress => "video-compress",
+            Tool::AudioExtract => "audio-extract",
+            Tool::VideoTrim => "video-trim",
+            Tool::GifMaker => "gif-maker",
+            Tool::AudioConvert => "audio-convert",
+        }
+    }
+
+    pub fn from_str(s: &str) -> Option<Self> {
+        match s {
+            "scan" => Some(Tool::Scan),
+            "image-compress" => Some(Tool::ImageCompress),
+            "image-resize" => Some(Tool::ImageResize),
+            "image-convert" => Some(Tool::ImageConvert),
+            "remove-bg" => Some(Tool::RemoveBg),
+            "pdf-merge" => Some(Tool::PdfMerge),
+            "pdf-split" => Some(Tool::PdfSplit),
+            "images-to-pdf" => Some(Tool::ImagesToPdf),
+            "pdf-compress" => Some(Tool::PdfCompress),
+            "pdf-to-images" => Some(Tool::PdfToImages),
+            "video-compress" => Some(Tool::VideoCompress),
+            "audio-extract" => Some(Tool::AudioExtract),
+            "video-trim" => Some(Tool::VideoTrim),
+            "gif-maker" => Some(Tool::GifMaker),
+            "audio-convert" => Some(Tool::AudioConvert),
+            _ => None,
+        }
+    }
+}
+
+/// Options for document scanning.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ScanOptions {
+    pub ocr: bool,
+    pub enhance: bool,
+    pub output_format: OutputFormat,
+    pub dpi: u32,
+    pub quality: u8,
+    pub language: String,
+    pub color_mode: ColorMode,
+    pub page_size: PageSize,
+}
+
+impl Default for ScanOptions {
+    fn default() -> Self {
+        Self {
+            ocr: true,
+            enhance: true,
+            output_format: OutputFormat::Pdf,
+            dpi: 300,
+            quality: 90,
+            language: "eng+ind".to_string(),
+            color_mode: ColorMode::BlackAndWhite,
+            page_size: PageSize::A4,
+        }
+    }
+}
+
+/// Options for image tools.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ImageOptions {
+    pub quality: Option<u8>,
+    pub width: Option<u32>,
+    pub height: Option<u32>,
+    pub format: Option<String>,
+    pub fit: Option<String>,
+    pub bg_color: Option<[u8; 3]>,
+}
+
+/// Options for PDF tools.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PdfOptions {
+    pub quality: Option<u8>,
+    pub pages: Option<String>,
+    pub dpi: Option<u32>,
+    pub page_size: Option<PageSize>,
+    pub margin_mm: Option<u32>,
+}
+
+/// A complete job record.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Job {
+    pub id: Uuid,
+    pub tool: Tool,
+    pub status: JobStatus,
+    pub file_path: String,
+    pub result_path: Option<String>,
+    pub file_size: u64,
+    pub options: serde_json::Value,
+    pub created_at: DateTime<Utc>,
+    pub ttl_seconds: u64,
+}
+
+/// Progress update sent via NATS and forwarded via WebSocket.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct JobProgress {
+    pub job_id: Uuid,
+    pub status: JobStatus,
+    pub stage: String,
+    pub progress: u8,
+    pub message: String,
+}
+
+/// Response returned after successful upload.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct UploadResponse {
+    pub job_id: Uuid,
+    pub status: String,
+    pub tool: String,
+    pub ws_url: String,
+    pub created_at: DateTime<Utc>,
+    pub estimated_seconds: u8,
+}
+
+/// Job status response.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct JobStatusResponse {
+    pub job_id: Uuid,
+    pub status: String,
+    pub tool: String,
+    pub progress: u8,
+    pub stage: String,
+    pub message: String,
+    pub result: Option<ResultInfo>,
+    pub created_at: DateTime<Utc>,
+    pub error: Option<String>,
+}
+
+/// Result metadata included in status response.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ResultInfo {
+    pub download_url: String,
+    pub file_size: u64,
+    pub file_name: String,
+    pub preview_url: Option<String>,
+}
+
+/// Output format for scan results.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum OutputFormat {
+    Pdf,
+    Jpeg,
+    Png,
+}
+
+/// Color mode for processed output.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum ColorMode {
+    BlackAndWhite,
+    Grayscale,
+    Color,
+}
+
+/// Page size for PDF output.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum PageSize {
+    A4,
+    Letter,
+    Auto,
+}