feat: add Streamable downloader endpoint + fix detect false-positive
Deploy Scraper / build-and-deploy (push) Canceled after 0s

- /download/streamable: real MP4 via yt-dlp (verified ftypisom)
- Fix detect_platform: t.co substring greedily matched reddit.com
  ('.reddit.com' contains 't.co') - now requires t.co/ path
- Removed duplicate reddit branch in detect chain
This commit is contained in:
asepharyana
2026-09-01 18:23:49 +07:00
parent 7850b5fa03
commit dfa37f86e3
4 changed files with 123 additions and 3 deletions
+9 -3
View File
@@ -115,6 +115,10 @@ pub async fn download_reddit(url: &str) -> Result<DownloadResult, ScrapingError>
DownloaderRepository::download_reddit(url).await DownloaderRepository::download_reddit(url).await
} }
pub async fn download_streamable(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_streamable(url).await
}
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> { pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
DownloaderRepository::download_bilibili(url).await DownloaderRepository::download_bilibili(url).await
} }
@@ -131,10 +135,12 @@ pub fn detect_platform(url: &str) -> String {
"youtube".to_string() "youtube".to_string()
} else if url.contains("open.spotify.com") || url.contains("spotify.link") { } else if url.contains("open.spotify.com") || url.contains("spotify.link") {
"spotify".to_string() "spotify".to_string()
} else if url.contains("twitter.com") || url.contains("x.com") || url.contains("t.co") { } else if url.contains("twitter.com") || url.contains("x.com") || url.contains("t.co/") {
"twitter".to_string() "twitter".to_string()
} else if url.contains("pinterest") { } else if url.contains("pinterest") {
"pinterest".to_string() "pinterest".to_string()
} else if url.contains("reddit.com") || url.contains("redd.it") {
"reddit".to_string()
} else if url.contains("mega.nz") || url.contains("mega.io") { } else if url.contains("mega.nz") || url.contains("mega.io") {
"mega".to_string() "mega".to_string()
} else if url.contains("terabox") || url.contains("nfile") { } else if url.contains("terabox") || url.contains("nfile") {
@@ -157,8 +163,8 @@ pub fn detect_platform(url: &str) -> String {
"soundcloud".to_string() "soundcloud".to_string()
} else if url.contains("dailymotion.com") { } else if url.contains("dailymotion.com") {
"dailymotion".to_string() "dailymotion".to_string()
} else if url.contains("reddit.com") || url.contains("redd.it") { } else if url.contains("streamable.com") {
"reddit".to_string() "streamable".to_string()
} else if url.contains("bilibili.com") || url.contains("b23.tv") { } else if url.contains("bilibili.com") || url.contains("b23.tv") {
"bilibili".to_string() "bilibili".to_string()
} else { } else {
+102
View File
@@ -470,6 +470,7 @@ impl DownloaderRepository {
"soundcloud" => Self::download_soundcloud(url).await, "soundcloud" => Self::download_soundcloud(url).await,
"dailymotion" => Self::download_dailymotion(url).await, "dailymotion" => Self::download_dailymotion(url).await,
"reddit" => Self::download_reddit(url).await, "reddit" => Self::download_reddit(url).await,
"streamable" => Self::download_streamable(url).await,
"bilibili" => Self::download_bilibili(url).await, "bilibili" => Self::download_bilibili(url).await,
_ => fetch_all_in_one(url).await, _ => fetch_all_in_one(url).await,
} }
@@ -602,6 +603,11 @@ impl DownloaderRepository {
fetch_reddit(url).await fetch_reddit(url).await
} }
/// Streamable video via yt-dlp.
pub async fn download_streamable(url: &str) -> Result<DownloadResult, ScrapingError> {
fetch_streamable(url).await
}
/// Bilibili video via b23.tv short link expansion. /// Bilibili video via b23.tv short link expansion.
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> { pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
fetch_bilibili(url).await fetch_bilibili(url).await
@@ -2401,6 +2407,102 @@ pub async fn fetch_reddit(url: &str) -> Result<DownloadResult, ScrapingError> {
Ok(result) Ok(result)
} }
// ============================================================================
// Streamable downloader (via yt-dlp)
// ============================================================================
pub async fn fetch_streamable(url: &str) -> Result<DownloadResult, ScrapingError> {
if !url.contains("streamable.com") {
return Err(ScrapingError::Http("Invalid Streamable URL".to_string()));
}
let data = run_ytdlp_json(url, &["-f", "best"]).await?;
let title = data
.get("title")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let thumbnail = data
.get("thumbnail")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let mut result = DownloadResult::success(title);
result.thumbnail = thumbnail;
result.provider = Some("yt-dlp".to_string());
let mut seen = std::collections::HashSet::new();
if let Some(fmts) = data.get("formats").and_then(|v| v.as_array()) {
for f in fmts {
if let (Some(furl), Some(ext_val)) = (
f.get("url").and_then(|v| v.as_str()),
f.get("ext").and_then(|v| v.as_str()),
) {
if ext_val == "mhtml" {
continue;
}
let url_string = furl.to_string();
if seen.insert(url_string.clone()) {
result.media.push(MediaItem {
url: url_string,
quality: f
.get("height")
.and_then(|v| v.as_u64())
.map(|h| format!("{}p", h)),
file_type: Some(MediaType::Video),
extension: Some(ext_val.to_string()),
thumbnail: data
.get("thumbnail")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
file_size: f
.get("filesize")
.and_then(|v| v.as_u64())
.map(format_filesize),
size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
frame_width: f
.get("width")
.and_then(|v| v.as_u64())
.map(|w| w.to_string()),
frame_height: f
.get("height")
.and_then(|v| v.as_u64())
.map(|h| h.to_string()),
note: None,
});
}
}
}
}
if result.media.is_empty() {
if let Some(dl_url) = data.get("url").and_then(|v| v.as_str()) {
result.media.push(MediaItem {
url: dl_url.to_string(),
quality: None,
file_type: Some(MediaType::Video),
extension: Some(
data.get("ext")
.and_then(|v| v.as_str())
.unwrap_or("mp4")
.to_string(),
),
thumbnail: data
.get("thumbnail")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
file_size: None,
size_bytes: None,
frame_width: None,
frame_height: None,
note: None,
});
}
}
Ok(result)
}
// ============================================================================ // ============================================================================
// Pinterest downloader // Pinterest downloader
// ============================================================================ // ============================================================================
+8
View File
@@ -211,6 +211,14 @@ pub async fn download_reddit(
Ok(Json(DownloadResponse::ok(result))) Ok(Json(DownloadResponse::ok(result)))
} }
/// Handler for _streamable.
pub async fn download_streamable(
Query(params): Query<DownloadParams>,
) -> Result<Json<DownloadResponse>, AppError> {
let result = use_cases::download_streamable(&params.url).await?;
Ok(Json(DownloadResponse::ok(result)))
}
/// Handler for _bilibili. /// Handler for _bilibili.
pub async fn download_bilibili( pub async fn download_bilibili(
Query(params): Query<DownloadParams>, Query(params): Query<DownloadParams>,
+4
View File
@@ -244,6 +244,10 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
"/download/reddit", "/download/reddit",
axum::routing::get(crate::presentation::handler::downloader::download_reddit), axum::routing::get(crate::presentation::handler::downloader::download_reddit),
) )
.route(
"/download/streamable",
axum::routing::get(crate::presentation::handler::downloader::download_streamable),
)
.route( .route(
"/download/bilibili", "/download/bilibili",
axum::routing::get(crate::presentation::handler::downloader::download_bilibili), axum::routing::get(crate::presentation::handler::downloader::download_bilibili),