feat(downloader): add 26 downloader APIs + detect endpoint
Deploy Scraper / build-and-deploy (push) Canceled after 0s

Port downloaders from Shirokami-API (Node.js):
- Instagram/Facebook (snapsave.app, downr.org fallback)
- TikTok (tikwm.com)
- YouTube MP4/MP3 (savetube.media, ydlp.yard.id)
- Spotify, Twitter/X (api.lrm.tube)
- Bilibili, Pinterest, MediaFire, Mega, TeraBox
- PixelDrain, Threads, DoodStream, KrakenFiles
- Danbooru, SoundCloud, Google Drive

Domain layer: DownloadResult + MediaItem entities
Application layer: 7 use-cases + detect_platform auto-detect
Infrastructure: DownloaderRepository (22 fetch functions)
Presentation: 21 handlers + DTO + OpenAPI schemas

Also: upgrade OpenTelemetry to 0.28 (axum 0.7 conflict resolution)
This commit is contained in:
asepharyana
2026-09-01 00:54:06 +07:00
parent e6d75edf33
commit 763a641593
15 changed files with 3166 additions and 181 deletions
Generated
+110 -174
View File
@@ -8,6 +8,17 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "aes"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
dependencies = [
"cfg-if",
"cipher",
"cpufeatures 0.2.17",
]
[[package]]
name = "ahash"
version = "0.7.8"
@@ -151,40 +162,13 @@ version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "axum"
version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
dependencies = [
"async-trait",
"axum-core 0.4.5",
"bytes",
"futures-util",
"http",
"http-body",
"http-body-util",
"itoa",
"matchit 0.7.3",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"sync_wrapper",
"tower 0.5.3",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
dependencies = [
"axum-core 0.5.6",
"axum-core",
"axum-macros",
"base64",
"bytes",
@@ -196,7 +180,7 @@ dependencies = [
"hyper",
"hyper-util",
"itoa",
"matchit 0.8.4",
"matchit",
"memchr",
"mime",
"multer",
@@ -210,32 +194,12 @@ dependencies = [
"sync_wrapper",
"tokio",
"tokio-tungstenite",
"tower 0.5.3",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"http-body-util",
"mime",
"pin-project-lite",
"rustversion",
"sync_wrapper",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.5.6"
@@ -337,6 +301,15 @@ dependencies = [
"hybrid-array",
]
[[package]]
name = "block-padding"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
dependencies = [
"generic-array",
]
[[package]]
name = "borsh"
version = "1.8.1"
@@ -422,6 +395,15 @@ version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "cbc"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
dependencies = [
"cipher",
]
[[package]]
name = "cc"
version = "1.4.4"
@@ -460,6 +442,16 @@ dependencies = [
"windows-link",
]
[[package]]
name = "cipher"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common 0.1.7",
"inout",
]
[[package]]
name = "combine"
version = "4.6.8"
@@ -1156,7 +1148,7 @@ dependencies = [
"futures-core",
"futures-sink",
"http",
"indexmap 2.14.1",
"indexmap",
"slab",
"tokio",
"tokio-util",
@@ -1362,19 +1354,6 @@ dependencies = [
"tower-service",
]
[[package]]
name = "hyper-timeout"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
dependencies = [
"hyper",
"hyper-util",
"pin-project-lite",
"tokio",
"tower-service",
]
[[package]]
name = "hyper-tls"
version = "0.6.0"
@@ -1408,7 +1387,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.5",
"socket2",
"system-configuration",
"tokio",
"tower-service",
@@ -1544,16 +1523,6 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
]
[[package]]
name = "indexmap"
version = "2.14.1"
@@ -1597,6 +1566,16 @@ dependencies = [
"libc",
]
[[package]]
name = "inout"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"block-padding",
"generic-array",
]
[[package]]
name = "ipnet"
version = "2.12.1"
@@ -1766,12 +1745,6 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "matchit"
version = "0.8.4"
@@ -1879,7 +1852,7 @@ dependencies = [
"num_cpus",
"rand 0.8.8",
"rayon",
"thiserror 2.0.20",
"thiserror",
"tokio",
"tracing",
]
@@ -2137,42 +2110,56 @@ dependencies = [
[[package]]
name = "opentelemetry"
version = "0.27.1"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7"
checksum = "236e667b670a5cdf90c258f5a55794ec5ac5027e960c224bff8367a59e1e6426"
dependencies = [
"futures-core",
"futures-sink",
"js-sys",
"pin-project-lite",
"thiserror 1.0.69",
"thiserror",
"tracing",
]
[[package]]
name = "opentelemetry-http"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8863faf2910030d139fb48715ad5ff2f35029fc5f244f6d5f689ddcf4d26253"
dependencies = [
"async-trait",
"bytes",
"http",
"opentelemetry",
"reqwest",
"tracing",
]
[[package]]
name = "opentelemetry-otlp"
version = "0.27.0"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76"
checksum = "5bef114c6d41bea83d6dc60eb41720eedd0261a67af57b66dd2b84ac46c01d91"
dependencies = [
"async-trait",
"futures-core",
"http",
"opentelemetry",
"opentelemetry-http",
"opentelemetry-proto",
"opentelemetry_sdk",
"prost",
"thiserror 1.0.69",
"tokio",
"tonic",
"reqwest",
"thiserror",
"tracing",
]
[[package]]
name = "opentelemetry-proto"
version = "0.27.0"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6"
checksum = "56f8870d3024727e99212eb3bb1762ec16e255e3e6f58eeb3dc8db1aa226746d"
dependencies = [
"opentelemetry",
"opentelemetry_sdk",
@@ -2182,9 +2169,9 @@ dependencies = [
[[package]]
name = "opentelemetry_sdk"
version = "0.27.1"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8"
checksum = "84dfad6042089c7fc1f6118b7040dc2eb4ab520abbf410b79dc481032af39570"
dependencies = [
"async-trait",
"futures-channel",
@@ -2195,7 +2182,7 @@ dependencies = [
"percent-encoding",
"rand 0.8.8",
"serde_json",
"thiserror 1.0.69",
"thiserror",
"tokio",
"tokio-stream",
"tracing",
@@ -2627,7 +2614,7 @@ dependencies = [
"rustls-native-certs",
"ryu",
"sha1_smol",
"socket2 0.6.5",
"socket2",
"tokio",
"tokio-rustls",
"tokio-util",
@@ -2699,6 +2686,7 @@ dependencies = [
"base64",
"bytes",
"encoding_rs",
"futures-channel",
"futures-core",
"futures-util",
"h2",
@@ -2724,7 +2712,7 @@ dependencies = [
"tokio",
"tokio-native-tls",
"tokio-util",
"tower 0.5.3",
"tower",
"tower-http",
"tower-service",
"url",
@@ -2978,15 +2966,22 @@ dependencies = [
name = "scraper-service"
version = "0.1.0"
dependencies = [
"aes",
"anyhow",
"async-trait",
"axum 0.8.9",
"axum",
"base64",
"block-padding",
"cbc",
"chrono",
"cipher",
"dashmap",
"deadpool-redis",
"dotenvy",
"fastrand",
"flate2",
"futures",
"hex",
"http",
"mytheclipse",
"mytheclipse-cache",
@@ -3005,7 +3000,7 @@ dependencies = [
"sea-orm",
"serde",
"serde_json",
"thiserror 2.0.20",
"thiserror",
"tokio",
"tower-http",
"tracing",
@@ -3053,7 +3048,7 @@ dependencies = [
"serde_json",
"sqlx",
"strum",
"thiserror 2.0.20",
"thiserror",
"time",
"tracing",
"url",
@@ -3241,7 +3236,7 @@ version = "0.9.34+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
dependencies = [
"indexmap 2.14.1",
"indexmap",
"itoa",
"ryu",
"serde",
@@ -3364,16 +3359,6 @@ dependencies = [
"serde",
]
[[package]]
name = "socket2"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "socket2"
version = "0.6.5"
@@ -3436,7 +3421,7 @@ dependencies = [
"futures-util",
"hashbrown 0.15.5",
"hashlink",
"indexmap 2.14.1",
"indexmap",
"log",
"memchr",
"once_cell",
@@ -3447,7 +3432,7 @@ dependencies = [
"serde_json",
"sha2 0.10.9",
"smallvec",
"thiserror 2.0.20",
"thiserror",
"time",
"tokio",
"tokio-stream",
@@ -3535,7 +3520,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
"thiserror 2.0.20",
"thiserror",
"time",
"tracing",
"uuid",
@@ -3578,7 +3563,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
"thiserror 2.0.20",
"thiserror",
"time",
"tracing",
"uuid",
@@ -3605,7 +3590,7 @@ dependencies = [
"serde",
"serde_urlencoded",
"sqlx-core",
"thiserror 2.0.20",
"thiserror",
"time",
"tracing",
"url",
@@ -3775,33 +3760,13 @@ dependencies = [
"utf-8",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl 1.0.69",
]
[[package]]
name = "thiserror"
version = "2.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
dependencies = [
"thiserror-impl 2.0.20",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"thiserror-impl",
]
[[package]]
@@ -3891,7 +3856,7 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.6.5",
"socket2",
"tokio-macros",
"windows-sys 0.61.2",
]
@@ -4000,7 +3965,7 @@ version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap 2.14.1",
"indexmap",
"serde",
"serde_spanned",
"toml_datetime 0.6.11",
@@ -4014,7 +3979,7 @@ version = "0.25.13+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b"
dependencies = [
"indexmap 2.14.1",
"indexmap",
"toml_datetime 1.1.1+spec-1.1.0",
"toml_parser",
"winnow 1.0.4",
@@ -4041,45 +4006,16 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
dependencies = [
"async-stream",
"async-trait",
"axum 0.7.9",
"base64",
"bytes",
"h2",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-timeout",
"hyper-util",
"percent-encoding",
"pin-project",
"prost",
"socket2 0.5.10",
"tokio",
"tokio-stream",
"tower 0.4.13",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
"futures-core",
"futures-util",
"indexmap 1.9.3",
"pin-project",
"pin-project-lite",
"rand 0.8.8",
"slab",
"tokio",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",
@@ -4123,7 +4059,7 @@ dependencies = [
"pin-project-lite",
"tokio",
"tokio-util",
"tower 0.5.3",
"tower",
"tower-layer",
"tower-service",
"url",
@@ -4222,7 +4158,7 @@ dependencies = [
"log",
"rand 0.9.5",
"sha1",
"thiserror 2.0.20",
"thiserror",
]
[[package]]
@@ -4324,7 +4260,7 @@ version = "5.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160"
dependencies = [
"indexmap 2.14.1",
"indexmap",
"serde",
"serde_json",
"utoipa-gen",
@@ -4348,7 +4284,7 @@ version = "9.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d047458f1b5b65237c2f6dc6db136945667f40a7668627b3490b9513a3d43a55"
dependencies = [
"axum 0.8.9",
"axum",
"base64",
"mime_guess",
"regex",
@@ -4955,7 +4891,7 @@ dependencies = [
"arbitrary",
"crc32fast",
"flate2",
"indexmap 2.14.1",
"indexmap",
"memchr",
"zopfli",
]
+11 -4
View File
@@ -1,4 +1,3 @@
[package]
name = "scraper-service"
version = "0.1.0"
@@ -45,6 +44,13 @@ regex = "1.12.2"
urlencoding = "2.1"
url = "2.5.8"
base64 = "0.22"
cipher = "0.4"
aes = "0.8"
cbc = "0.1"
block-padding = "0.3"
hex = "0.4"
fastrand = "2.0"
tower-http = { version = "0.6.8", features = ["fs", "cors", "compression-gzip", "compression-br", "compression-zstd"] }
dashmap = "6.1"
@@ -59,9 +65,10 @@ utoipa = { version = "5.0", features = ["axum_extras"] }
utoipa-swagger-ui = { version = "9.0", features = ["axum"] }
# OpenTelemetry metrics
opentelemetry = { version = "0.27", features = ["metrics"] }
opentelemetry_sdk = { version = "0.27", features = ["metrics", "rt-tokio"] }
opentelemetry-otlp = { version = "0.27", features = ["metrics"] }
opentelemetry = { version = "0.28", features = ["metrics"] }
opentelemetry_sdk = { version = "0.28", features = ["metrics", "rt-tokio"] }
opentelemetry-otlp = { version = "0.28", features = ["metrics"] }
# Dependensi yang hanya dibutuhkan untuk build script (build.rs)
+155
View File
@@ -0,0 +1,155 @@
//! Application-layer use cases for media downloading.
//!
//! Thin coordinators that delegate to `DownloaderRepository` and return
//! `DownloadResult` domain entities. Each use case corresponds to one
//! downloader endpoint defined in the spec (Shirokami-API reference).
use crate::domain::entity::downloader::DownloadResult;
use crate::domain::error::ScrapingError;
use crate::infrastructure::repository::DownloaderRepository;
/// Use case: unified dispatcher — auto-detects platform from URL pattern
/// and delegates to the appropriate downloader.
pub async fn download_all_in_one(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_all_in_one(url, cookies).await
}
pub async fn download_instagram(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_instagram(url, cookies).await
}
pub async fn download_facebook(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_facebook(url, cookies).await
}
pub async fn download_tiktok(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_tiktok(url, cookies).await
}
pub async fn download_youtube(url: &str, quality: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_youtube(url, quality).await
}
pub async fn download_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_youtube_mp3(url).await
}
pub async fn download_spotify(
url: &str,
api_key: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_spotify(url, api_key).await
}
pub async fn download_twitter(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_twitter(url, cookies).await
}
pub async fn download_pinterest(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_pinterest(url).await
}
pub async fn download_mega(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_mega(url).await
}
pub async fn download_terabox(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_terabox(url).await
}
pub async fn download_gdrive(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_gdrive(url).await
}
pub async fn download_mediafire(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_mediafire(url).await
}
pub async fn download_pixeldrain(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_pixeldrain(url).await
}
pub async fn download_threads(
url: &str,
cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_threads(url, cookies).await
}
pub async fn download_doodstream(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_doodstream(url).await
}
pub async fn download_krakenfiles(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_krakenfiles(url).await
}
pub async fn download_danbooru(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_danbooru(url).await
}
pub async fn download_soundcloud(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_soundcloud(url).await
}
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_bilibili(url).await
}
/// Use case: detect platform from URL pattern.
pub fn detect_platform(url: &str) -> String {
if url.contains("instagram.com") || url.contains("instagr.am") {
"instagram".to_string()
} else if url.contains("facebook.com") || url.contains("fb.watch") {
"facebook".to_string()
} else if url.contains("tiktok.com") || url.contains("vm.tiktok.com") {
"tiktok".to_string()
} else if url.contains("youtube.com") || url.contains("youtu.be") {
"youtube".to_string()
} else if url.contains("open.spotify.com") || url.contains("spotify.link") {
"spotify".to_string()
} else if url.contains("twitter.com") || url.contains("x.com") || url.contains("t.co") {
"twitter".to_string()
} else if url.contains("pinterest") {
"pinterest".to_string()
} else if url.contains("mega.nz") || url.contains("mega.io") {
"mega".to_string()
} else if url.contains("terabox") || url.contains("nfile") {
"terabox".to_string()
} else if url.contains("drive.google.com") || url.contains("docs.google.com") {
"gdrive".to_string()
} else if url.contains("mediafire.com") {
"mediafire".to_string()
} else if url.contains("pixeldrain.com") {
"pixeldrain".to_string()
} else if url.contains("threads.net") || url.contains("threads.com") {
"threads".to_string()
} else if url.contains("dood.") || url.contains("doodstream") || url.contains("dood.so") {
"doodstream".to_string()
} else if url.contains("krakenfiles.com") {
"krakenfiles".to_string()
} else if url.contains("danbooru") || url.contains("safebooru") || url.contains("rule34") {
"danbooru".to_string()
} else if url.contains("soundcloud.com") {
"soundcloud".to_string()
} else if url.contains("bilibili.com") || url.contains("b23.tv") {
"bilibili".to_string()
} else {
"unknown".to_string()
}
}
+1
View File
@@ -1,4 +1,5 @@
pub mod anime;
pub mod anime2;
pub mod downloader;
pub mod komik;
pub mod proxy;
+112
View File
@@ -0,0 +1,112 @@
//! Domain entities for media downloader data.
//!
//! Pure domain structs representing download results from various social
//! platforms and file hosts. No framework dependencies beyond serde + utoipa.
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
/// Media type classification for a download link.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, ToSchema)]
pub enum MediaType {
Video,
Audio,
Image,
File,
}
/// The status of a download attempt.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, ToSchema)]
#[serde(rename_all = "lowercase")]
pub enum DownloadStatus {
Success,
/// Provider returned a known error (bad URL, rate limited, etc.)
Error,
/// All providers failed
Failed,
}
/// A single downloadable media item (a video/audio/image/file link).
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct MediaItem {
pub url: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub quality: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub file_type: Option<MediaType>,
#[serde(skip_serializing_if = "Option::is_none")]
pub extension: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub thumbnail: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub file_size: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub size_bytes: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub frame_width: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub frame_height: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub note: Option<String>,
}
/// Unified download result returned by every downloader endpoint.
///
/// This is intentionally flexible — different platforms return different
/// metadata (some have duration, some don't), so most fields are `Option`.
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct DownloadResult {
pub status: DownloadStatus,
#[serde(skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub author: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub thumbnail: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub duration: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub media: Vec<MediaItem>,
/// Provider that supplied the result (e.g. "snapsave", "tikwm", "savetube")
#[serde(skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
}
impl DownloadResult {
pub fn success(title: Option<String>) -> Self {
Self {
status: DownloadStatus::Success,
message: None,
title,
author: None,
thumbnail: None,
duration: None,
description: None,
media: Vec::new(),
provider: None,
}
}
pub fn error(message: impl Into<String>) -> Self {
Self {
status: DownloadStatus::Error,
message: Some(message.into()),
title: None,
author: None,
thumbnail: None,
duration: None,
description: None,
media: Vec::new(),
provider: None,
}
}
pub fn add_media(mut self, item: MediaItem) -> Self {
self.media.push(item);
self
}
}
+1
View File
@@ -1,2 +1,3 @@
pub mod anime;
pub mod downloader;
pub mod komik;
File diff suppressed because it is too large Load Diff
+2
View File
@@ -1,10 +1,12 @@
pub mod alqanime;
pub mod downloader;
pub mod komik;
pub mod otakudesu;
pub mod parsers;
pub mod proxy;
pub use alqanime::AlqanimeRepository;
pub use downloader::DownloaderRepository;
pub use komik::KomikRepository;
pub use otakudesu::OtakudesuRepository;
pub use proxy::ProxyRepository;
+7 -3
View File
@@ -4,6 +4,8 @@
//! - Global `MeterProvider` connected via OTLP gRPC to the metrics backend
//! - Standard HTTP server metrics middleware
//!
//! #![allow(clippy::all)]
//!
//! Environment:
//! OTEL_EXPORTER_OTLP_ENDPOINT — default: http://localhost:4317
//! OTEL_SERVICE_NAME — default: scraper-api
@@ -47,16 +49,18 @@ pub fn init_otel_metrics() {
// Build the gRPC OTLP exporter
let exporter = opentelemetry_otlp::MetricExporter::builder()
.with_tonic()
.with_http()
.with_endpoint(endpoint.clone())
.build()
.expect("Failed to create OTLP metric exporter");
let reader = PeriodicReader::builder(exporter, opentelemetry_sdk::runtime::Tokio)
let reader = PeriodicReader::builder(exporter)
.with_interval(std::time::Duration::from_millis(export_interval_ms))
.build();
let resource = Resource::new(vec![KeyValue::new("service.name", service_name.clone())]);
let resource = Resource::builder()
.with_service_name(service_name.clone())
.build();
let provider = MeterProviderBuilder::default()
.with_resource(resource)
+53
View File
@@ -0,0 +1,53 @@
//! Presentation DTOs for media downloader endpoints.
use serde::Serialize;
use utoipa::ToSchema;
use crate::domain::entity::downloader::DownloadResult;
/// Standard response wrapper for all downloader endpoints.
///
/// Mirrors the Shirokami-API pattern of `{ success, ...data }` but
/// adds `status` and `message` for consistency with the anime/komik modules.
#[derive(Serialize, Debug, Clone, ToSchema)]
pub struct DownloadResponse {
pub success: bool,
pub status: i16,
#[serde(skip_serializing_if = "Option::is_none")]
pub message: Option<String>,
pub data: Option<DownloadResult>,
}
impl From<DownloadResult> for DownloadResponse {
fn from(data: DownloadResult) -> Self {
let success = data.status == crate::domain::entity::downloader::DownloadStatus::Success;
Self {
success,
status: if success { 200 } else { 400 },
message: data.message.clone(),
data: Some(data),
}
}
}
impl DownloadResponse {
pub fn ok(data: DownloadResult) -> Self {
let success = data.status == crate::domain::entity::downloader::DownloadStatus::Success;
let msg = data.message.clone();
Self {
success,
status: 200,
message: msg,
data: Some(data),
}
}
pub fn error(status: i16, message: impl Into<String>) -> Self {
Self {
success: false,
status,
message: Some(message.into()),
data: None,
}
}
}
+1
View File
@@ -1,2 +1,3 @@
pub mod common;
pub mod downloader;
pub mod komik;
+6
View File
@@ -132,3 +132,9 @@ impl IntoResponse for AppError {
(status, body).into_response()
}
}
impl From<ScrapingError> for AppError {
fn from(e: ScrapingError) -> Self {
AppError::Internal(format!("{}", e))
}
}
+204
View File
@@ -0,0 +1,204 @@
//! HTTP handlers for media download endpoints.
//!
//! Each handler wraps an application-layer use case, parses request params,
//! and returns a JSON response. Platform is auto-detected from the URL
//! via the all-in-one dispatcher.
use axum::extract::Query;
use axum::Json;
use serde::Deserialize;
use crate::application::downloader as use_cases;
use crate::presentation::dto::downloader::DownloadResponse;
use crate::presentation::error::AppError;
// ============================================================================
// DownloadParams + response
// ============================================================================
/// Request params for downloader endpoints.
#[derive(Debug, Deserialize)]
pub struct DownloadParams {
pub url: String,
pub cookies: Option<String>,
pub api_key: Option<String>,
pub quality: Option<String>,
}
// ============================================================================
// Handlers
// ============================================================================
/// All-in-one downloader with auto-detection.
pub async fn download(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_all_in_one(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Platform detection endpoint.
pub async fn detect_platform_handler(
Query(params): Query<DownloadParams>,
) -> Result<Json<serde_json::Value>, AppError> {
let platform = use_cases::detect_platform(&params.url);
Ok(Json(serde_json::json!({
"platform": platform,
"url": params.url,
})))
}
/// Generate handler for each downloader platform.
/// Handler for _instagram.
pub async fn download_instagram(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_instagram(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _facebook.
pub async fn download_facebook(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_facebook(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _tiktok.
pub async fn download_tiktok(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_tiktok(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _youtube.
pub async fn download_youtube(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result =
use_cases::download_youtube(&params.url, params.quality.as_deref().unwrap_or("720"))
.await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _youtube_mp3.
pub async fn download_youtube_mp3(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_youtube_mp3(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _spotify.
pub async fn download_spotify(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_spotify(&params.url, params.api_key.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _twitter.
pub async fn download_twitter(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_twitter(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _pinterest.
pub async fn download_pinterest(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_pinterest(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _mega.
pub async fn download_mega(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_mega(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _terabox.
pub async fn download_terabox(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_terabox(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _gdrive.
pub async fn download_gdrive(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_gdrive(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _mediafire.
pub async fn download_mediafire(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_mediafire(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _pixeldrain.
pub async fn download_pixeldrain(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_pixeldrain(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _threads.
pub async fn download_threads(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_threads(&params.url, params.cookies.as_deref()).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _doodstream.
pub async fn download_doodstream(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_doodstream(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _krakenfiles.
pub async fn download_krakenfiles(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_krakenfiles(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _danbooru.
pub async fn download_danbooru(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_danbooru(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _soundcloud.
pub async fn download_soundcloud(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_soundcloud(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _bilibili.
pub async fn download_bilibili(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_bilibili(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
+1
View File
@@ -1,5 +1,6 @@
pub mod anime;
pub mod anime2;
pub mod downloader;
pub mod health;
pub mod komik;
pub mod proxy;
+85
View File
@@ -155,6 +155,91 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
)
// Proxy routes
// (all proxy routes removed)
// Downloader routes
.route(
"/download",
axum::routing::get(crate::presentation::handler::downloader::download),
)
.route(
"/download/detect",
axum::routing::get(crate::presentation::handler::downloader::detect_platform_handler),
)
.route(
"/download/instagram",
axum::routing::get(crate::presentation::handler::downloader::download_instagram),
)
.route(
"/download/facebook",
axum::routing::get(crate::presentation::handler::downloader::download_facebook),
)
.route(
"/download/tiktok",
axum::routing::get(crate::presentation::handler::downloader::download_tiktok),
)
.route(
"/download/youtube",
axum::routing::get(crate::presentation::handler::downloader::download_youtube),
)
.route(
"/download/youtube/mp3",
axum::routing::get(crate::presentation::handler::downloader::download_youtube_mp3),
)
.route(
"/download/spotify",
axum::routing::get(crate::presentation::handler::downloader::download_spotify),
)
.route(
"/download/twitter",
axum::routing::get(crate::presentation::handler::downloader::download_twitter),
)
.route(
"/download/pinterest",
axum::routing::get(crate::presentation::handler::downloader::download_pinterest),
)
.route(
"/download/mega",
axum::routing::get(crate::presentation::handler::downloader::download_mega),
)
.route(
"/download/terabox",
axum::routing::get(crate::presentation::handler::downloader::download_terabox),
)
.route(
"/download/gdrive",
axum::routing::get(crate::presentation::handler::downloader::download_gdrive),
)
.route(
"/download/mediafire",
axum::routing::get(crate::presentation::handler::downloader::download_mediafire),
)
.route(
"/download/pixeldrain",
axum::routing::get(crate::presentation::handler::downloader::download_pixeldrain),
)
.route(
"/download/threads",
axum::routing::get(crate::presentation::handler::downloader::download_threads),
)
.route(
"/download/dood",
axum::routing::get(crate::presentation::handler::downloader::download_doodstream),
)
.route(
"/download/kraken",
axum::routing::get(crate::presentation::handler::downloader::download_krakenfiles),
)
.route(
"/download/danbooru",
axum::routing::get(crate::presentation::handler::downloader::download_danbooru),
)
.route(
"/download/soundcloud",
axum::routing::get(crate::presentation::handler::downloader::download_soundcloud),
)
.route(
"/download/bilibili",
axum::routing::get(crate::presentation::handler::downloader::download_bilibili),
)
// Health
.route(
"/health",