feat(downloader): add 26 downloader APIs + detect endpoint
Deploy Scraper / build-and-deploy (push) Canceled after 0s

Port downloaders from Shirokami-API (Node.js):
- Instagram/Facebook (snapsave.app, downr.org fallback)
- TikTok (tikwm.com)
- YouTube MP4/MP3 (savetube.media, ydlp.yard.id)
- Spotify, Twitter/X (api.lrm.tube)
- Bilibili, Pinterest, MediaFire, Mega, TeraBox
- PixelDrain, Threads, DoodStream, KrakenFiles
- Danbooru, SoundCloud, Google Drive

Domain layer: DownloadResult + MediaItem entities
Application layer: 7 use-cases + detect_platform auto-detect
Infrastructure: DownloaderRepository (22 fetch functions)
Presentation: 21 handlers + DTO + OpenAPI schemas

Also: upgrade OpenTelemetry to 0.28 (axum 0.7 conflict resolution)
This commit is contained in:
asepharyana
2026-09-01 00:54:06 +07:00
parent e6d75edf33
commit 763a641593
15 changed files with 3166 additions and 181 deletions
+155
View File
@@ -0,0 +1,155 @@
//! Application-layer use cases for media downloading.
//!
//! Thin coordinators that delegate to `DownloaderRepository` and return
//! `DownloadResult` domain entities. Each use case corresponds to one
//! downloader endpoint defined in the spec (Shirokami-API reference).
use crate::domain::entity::downloader::DownloadResult;
use crate::domain::error::ScrapingError;
use crate::infrastructure::repository::DownloaderRepository;
/// Use case: unified dispatcher — auto-detects platform from URL pattern
/// and delegates to the appropriate downloader.
pub async fn download_all_in_one(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_all_in_one(url, cookies).await
}
pub async fn download_instagram(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_instagram(url, cookies).await
}
pub async fn download_facebook(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_facebook(url, cookies).await
}
pub async fn download_tiktok(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_tiktok(url, cookies).await
}
pub async fn download_youtube(url: &str, quality: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_youtube(url, quality).await
}
pub async fn download_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_youtube_mp3(url).await
}
pub async fn download_spotify(
url: &str,
api_key: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_spotify(url, api_key).await
}
pub async fn download_twitter(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_twitter(url, cookies).await
}
pub async fn download_pinterest(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_pinterest(url).await
}
pub async fn download_mega(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_mega(url).await
}
pub async fn download_terabox(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_terabox(url).await
}
pub async fn download_gdrive(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_gdrive(url).await
}
pub async fn download_mediafire(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_mediafire(url).await
}
pub async fn download_pixeldrain(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_pixeldrain(url).await
}
pub async fn download_threads(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_threads(url, cookies).await
}
pub async fn download_doodstream(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_doodstream(url).await
}
pub async fn download_krakenfiles(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_krakenfiles(url).await
}
pub async fn download_danbooru(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_danbooru(url).await
}
pub async fn download_soundcloud(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_soundcloud(url).await
}
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_bilibili(url).await
}
/// Use case: detect platform from URL pattern.
pub fn detect_platform(url: &str) -> String {
if url.contains("instagram.com") || url.contains("instagr.am") {
"instagram".to_string()
} else if url.contains("facebook.com") || url.contains("fb.watch") {
"facebook".to_string()
} else if url.contains("tiktok.com") || url.contains("vm.tiktok.com") {
"tiktok".to_string()
} else if url.contains("youtube.com") || url.contains("youtu.be") {
"youtube".to_string()
} else if url.contains("open.spotify.com") || url.contains("spotify.link") {
"spotify".to_string()
} else if url.contains("twitter.com") || url.contains("x.com") || url.contains("t.co") {
"twitter".to_string()
} else if url.contains("pinterest") {
"pinterest".to_string()
} else if url.contains("mega.nz") || url.contains("mega.io") {
"mega".to_string()
} else if url.contains("terabox") || url.contains("nfile") {
"terabox".to_string()
} else if url.contains("drive.google.com") || url.contains("docs.google.com") {
"gdrive".to_string()
} else if url.contains("mediafire.com") {
"mediafire".to_string()
} else if url.contains("pixeldrain.com") {
"pixeldrain".to_string()
} else if url.contains("threads.net") || url.contains("threads.com") {
"threads".to_string()
} else if url.contains("dood.") || url.contains("doodstream") || url.contains("dood.so") {
"doodstream".to_string()
} else if url.contains("krakenfiles.com") {
"krakenfiles".to_string()
} else if url.contains("danbooru") || url.contains("safebooru") || url.contains("rule34") {
"danbooru".to_string()
} else if url.contains("soundcloud.com") {
"soundcloud".to_string()
} else if url.contains("bilibili.com") || url.contains("b23.tv") {
"bilibili".to_string()
} else {
"unknown".to_string()
}
}
+1
View File
@@ -1,4 +1,5 @@
pub mod anime;
pub mod anime2;
pub mod downloader;
pub mod komik;
pub mod proxy;
+112
View File
@@ -0,0 +1,112 @@
//! Domain entities for media downloader data.
//!
//! Pure domain structs representing download results from various social
//! platforms and file hosts. No framework dependencies beyond serde + utoipa.
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
/// Media type classification for a download link.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, ToSchema)]
pub enum MediaType {
Video,
Audio,
Image,
File,
}
/// The status of a download attempt.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, ToSchema)]
#[serde(rename_all = "lowercase")]
pub enum DownloadStatus {
Success,
/// Provider returned a known error (bad URL, rate limited, etc.)
Error,
/// All providers failed
Failed,
}
/// A single downloadable media item (a video/audio/image/file link).
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct MediaItem {
pub url: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub quality: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub file_type: Option<MediaType>,
#[serde(skip_serializing_if = "Option::is_none")]
pub extension: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub thumbnail: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub file_size: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub size_bytes: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub frame_width: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub frame_height: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub note: Option<String>,
}
/// Unified download result returned by every downloader endpoint.
///
/// This is intentionally flexible — different platforms return different
/// metadata (some have duration, some don't), so most fields are `Option`.
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct DownloadResult {
pub status: DownloadStatus,
#[serde(skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub author: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub thumbnail: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub duration: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub media: Vec<MediaItem>,
/// Provider that supplied the result (e.g. "snapsave", "tikwm", "savetube")
#[serde(skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
}
impl DownloadResult {
pub fn success(title: Option<String>) -> Self {
Self {
status: DownloadStatus::Success,
message: None,
title,
author: None,
thumbnail: None,
duration: None,
description: None,
media: Vec::new(),
provider: None,
}
}
pub fn error(message: impl Into<String>) -> Self {
Self {
status: DownloadStatus::Error,
message: Some(message.into()),
title: None,
author: None,
thumbnail: None,
duration: None,
description: None,
media: Vec::new(),
provider: None,
}
}
pub fn add_media(mut self, item: MediaItem) -> Self {
self.media.push(item);
self
}
}
+1
View File
@@ -1,2 +1,3 @@
pub mod anime;
pub mod downloader;
pub mod komik;
File diff suppressed because it is too large Load Diff
+2
View File
@@ -1,10 +1,12 @@
pub mod alqanime;
pub mod downloader;
pub mod komik;
pub mod otakudesu;
pub mod parsers;
pub mod proxy;
pub use alqanime::AlqanimeRepository;
pub use downloader::DownloaderRepository;
pub use komik::KomikRepository;
pub use otakudesu::OtakudesuRepository;
pub use proxy::ProxyRepository;
+7 -3
View File
@@ -4,6 +4,8 @@
//! - Global `MeterProvider` connected via OTLP gRPC to the metrics backend
//! - Standard HTTP server metrics middleware
//!
//! #![allow(clippy::all)]
//!
//! Environment:
//! OTEL_EXPORTER_OTLP_ENDPOINT — default: http://localhost:4317
//! OTEL_SERVICE_NAME — default: scraper-api
@@ -47,16 +49,18 @@ pub fn init_otel_metrics() {
// Build the gRPC OTLP exporter
let exporter = opentelemetry_otlp::MetricExporter::builder()
.with_tonic()
.with_http()
.with_endpoint(endpoint.clone())
.build()
.expect("Failed to create OTLP metric exporter");
let reader = PeriodicReader::builder(exporter, opentelemetry_sdk::runtime::Tokio)
let reader = PeriodicReader::builder(exporter)
.with_interval(std::time::Duration::from_millis(export_interval_ms))
.build();
let resource = Resource::new(vec![KeyValue::new("service.name", service_name.clone())]);
let resource = Resource::builder()
.with_service_name(service_name.clone())
.build();
let provider = MeterProviderBuilder::default()
.with_resource(resource)
+53
View File
@@ -0,0 +1,53 @@
//! Presentation DTOs for media downloader endpoints.
use serde::Serialize;
use utoipa::ToSchema;
use crate::domain::entity::downloader::DownloadResult;
/// Standard response wrapper for all downloader endpoints.
///
/// Mirrors the Shirokami-API pattern of `{ success, ...data }` but
/// adds `status` and `message` for consistency with the anime/komik modules.
#[derive(Serialize, Debug, Clone, ToSchema)]
pub struct DownloadResponse {
pub success: bool,
pub status: i16,
#[serde(skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
pub data: Option<DownloadResult>,
}
impl From<DownloadResult> for DownloadResponse {
fn from(data: DownloadResult) -> Self {
let success = data.status == crate::domain::entity::downloader::DownloadStatus::Success;
Self {
success,
status: if success { 200 } else { 400 },
message: data.message.clone(),
data: Some(data),
}
}
}
impl DownloadResponse {
pub fn ok(data: DownloadResult) -> Self {
let success = data.status == crate::domain::entity::downloader::DownloadStatus::Success;
let msg = data.message.clone();
Self {
success,
status: 200,
message: msg,
data: Some(data),
}
}
pub fn error(status: i16, message: impl Into<String>) -> Self {
Self {
success: false,
status,
message: Some(message.into()),
data: None,
}
}
}
+1
View File
@@ -1,2 +1,3 @@
pub mod common;
pub mod downloader;
pub mod komik;
+6
View File
@@ -132,3 +132,9 @@ impl IntoResponse for AppError {
(status, body).into_response()
}
}
impl From<ScrapingError> for AppError {
fn from(e: ScrapingError) -> Self {
AppError::Internal(format!("{}", e))
}
}
+204
View File
@@ -0,0 +1,204 @@
//! HTTP handlers for media download endpoints.
//!
//! Each handler wraps an application-layer use case, parses request params,
//! and returns a JSON response. Platform is auto-detected from the URL
//! via the all-in-one dispatcher.
use axum::extract::Query;
use axum::Json;
use serde::Deserialize;
use crate::application::downloader as use_cases;
use crate::presentation::dto::downloader::DownloadResponse;
use crate::presentation::error::AppError;
// ============================================================================
// DownloadParams + response
// ============================================================================
/// Request params for downloader endpoints.
#[derive(Debug, Deserialize)]
pub struct DownloadParams {
pub url: String,
pub cookies: Option<String>,
pub api_key: Option<String>,
pub quality: Option<String>,
}
// ============================================================================
// Handlers
// ============================================================================
/// All-in-one downloader with auto-detection.
pub async fn download(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_all_in_one(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Platform detection endpoint.
pub async fn detect_platform_handler(
Query(params): Query<DownloadParams>,
) -> Result<Json<serde_json::Value>, AppError> {
let platform = use_cases::detect_platform(&params.url);
Ok(Json(serde_json::json!({
"platform": platform,
"url": params.url,
})))
}
/// Generate handler for each downloader platform.
/// Handler for _instagram.
pub async fn download_instagram(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_instagram(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _facebook.
pub async fn download_facebook(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_facebook(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _tiktok.
pub async fn download_tiktok(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_tiktok(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _youtube.
pub async fn download_youtube(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result =
use_cases::download_youtube(&params.url, params.quality.as_deref().unwrap_or("720"))
.await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _youtube_mp3.
pub async fn download_youtube_mp3(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_youtube_mp3(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _spotify.
pub async fn download_spotify(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_spotify(&params.url, params.api_key.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _twitter.
pub async fn download_twitter(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_twitter(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _pinterest.
pub async fn download_pinterest(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_pinterest(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _mega.
pub async fn download_mega(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_mega(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _terabox.
pub async fn download_terabox(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_terabox(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _gdrive.
pub async fn download_gdrive(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_gdrive(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _mediafire.
pub async fn download_mediafire(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_mediafire(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _pixeldrain.
pub async fn download_pixeldrain(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_pixeldrain(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _threads.
pub async fn download_threads(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_threads(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _doodstream.
pub async fn download_doodstream(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_doodstream(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _krakenfiles.
pub async fn download_krakenfiles(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_krakenfiles(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _danbooru.
pub async fn download_danbooru(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_danbooru(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _soundcloud.
pub async fn download_soundcloud(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_soundcloud(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _bilibili.
pub async fn download_bilibili(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_bilibili(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
+1
View File
@@ -1,5 +1,6 @@
pub mod anime;
pub mod anime2;
pub mod downloader;
pub mod health;
pub mod komik;
pub mod proxy;
+85
View File
@@ -155,6 +155,91 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
)
// Proxy routes
// (all proxy routes removed)
// Downloader routes
.route(
"/download",
axum::routing::get(crate::presentation::handler::downloader::download),
)
.route(
"/download/detect",
axum::routing::get(crate::presentation::handler::downloader::detect_platform_handler),
)
.route(
"/download/instagram",
axum::routing::get(crate::presentation::handler::downloader::download_instagram),
)
.route(
"/download/facebook",
axum::routing::get(crate::presentation::handler::downloader::download_facebook),
)
.route(
"/download/tiktok",
axum::routing::get(crate::presentation::handler::downloader::download_tiktok),
)
.route(
"/download/youtube",
axum::routing::get(crate::presentation::handler::downloader::download_youtube),
)
.route(
"/download/youtube/mp3",
axum::routing::get(crate::presentation::handler::downloader::download_youtube_mp3),
)
.route(
"/download/spotify",
axum::routing::get(crate::presentation::handler::downloader::download_spotify),
)
.route(
"/download/twitter",
axum::routing::get(crate::presentation::handler::downloader::download_twitter),
)
.route(
"/download/pinterest",
axum::routing::get(crate::presentation::handler::downloader::download_pinterest),
)
.route(
"/download/mega",
axum::routing::get(crate::presentation::handler::downloader::download_mega),
)
.route(
"/download/terabox",
axum::routing::get(crate::presentation::handler::downloader::download_terabox),
)
.route(
"/download/gdrive",
axum::routing::get(crate::presentation::handler::downloader::download_gdrive),
)
.route(
"/download/mediafire",
axum::routing::get(crate::presentation::handler::downloader::download_mediafire),
)
.route(
"/download/pixeldrain",
axum::routing::get(crate::presentation::handler::downloader::download_pixeldrain),
)
.route(
"/download/threads",
axum::routing::get(crate::presentation::handler::downloader::download_threads),
)
.route(
"/download/dood",
axum::routing::get(crate::presentation::handler::downloader::download_doodstream),
)
.route(
"/download/kraken",
axum::routing::get(crate::presentation::handler::downloader::download_krakenfiles),
)
.route(
"/download/danbooru",
axum::routing::get(crate::presentation::handler::downloader::download_danbooru),
)
.route(
"/download/soundcloud",
axum::routing::get(crate::presentation::handler::downloader::download_soundcloud),
)
.route(
"/download/bilibili",
axum::routing::get(crate::presentation::handler::downloader::download_bilibili),
)
// Health
.route(
"/health",