feat: add Streamable downloader endpoint + fix detect false-positive
Deploy Scraper / build-and-deploy (push) Canceled after 0s
Deploy Scraper / build-and-deploy (push) Canceled after 0s
- /download/streamable: real MP4 via yt-dlp (verified ftypisom)
- Fix detect_platform: t.co substring greedily matched reddit.com
('.reddit.com' contains 't.co') - now requires t.co/ path
- Removed duplicate reddit branch in detect chain
This commit is contained in:
@@ -115,6 +115,10 @@ pub async fn download_reddit(url: &str) -> Result<DownloadResult, ScrapingError>
|
||||
DownloaderRepository::download_reddit(url).await
|
||||
}
|
||||
|
||||
pub async fn download_streamable(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
DownloaderRepository::download_streamable(url).await
|
||||
}
|
||||
|
||||
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
DownloaderRepository::download_bilibili(url).await
|
||||
}
|
||||
@@ -131,10 +135,12 @@ pub fn detect_platform(url: &str) -> String {
|
||||
"youtube".to_string()
|
||||
} else if url.contains("open.spotify.com") || url.contains("spotify.link") {
|
||||
"spotify".to_string()
|
||||
} else if url.contains("twitter.com") || url.contains("x.com") || url.contains("t.co") {
|
||||
} else if url.contains("twitter.com") || url.contains("x.com") || url.contains("t.co/") {
|
||||
"twitter".to_string()
|
||||
} else if url.contains("pinterest") {
|
||||
"pinterest".to_string()
|
||||
} else if url.contains("reddit.com") || url.contains("redd.it") {
|
||||
"reddit".to_string()
|
||||
} else if url.contains("mega.nz") || url.contains("mega.io") {
|
||||
"mega".to_string()
|
||||
} else if url.contains("terabox") || url.contains("nfile") {
|
||||
@@ -157,8 +163,8 @@ pub fn detect_platform(url: &str) -> String {
|
||||
"soundcloud".to_string()
|
||||
} else if url.contains("dailymotion.com") {
|
||||
"dailymotion".to_string()
|
||||
} else if url.contains("reddit.com") || url.contains("redd.it") {
|
||||
"reddit".to_string()
|
||||
} else if url.contains("streamable.com") {
|
||||
"streamable".to_string()
|
||||
} else if url.contains("bilibili.com") || url.contains("b23.tv") {
|
||||
"bilibili".to_string()
|
||||
} else {
|
||||
|
||||
@@ -470,6 +470,7 @@ impl DownloaderRepository {
|
||||
"soundcloud" => Self::download_soundcloud(url).await,
|
||||
"dailymotion" => Self::download_dailymotion(url).await,
|
||||
"reddit" => Self::download_reddit(url).await,
|
||||
"streamable" => Self::download_streamable(url).await,
|
||||
"bilibili" => Self::download_bilibili(url).await,
|
||||
_ => fetch_all_in_one(url).await,
|
||||
}
|
||||
@@ -602,6 +603,11 @@ impl DownloaderRepository {
|
||||
fetch_reddit(url).await
|
||||
}
|
||||
|
||||
/// Streamable video via yt-dlp.
|
||||
pub async fn download_streamable(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
fetch_streamable(url).await
|
||||
}
|
||||
|
||||
/// Bilibili video via b23.tv short link expansion.
|
||||
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
fetch_bilibili(url).await
|
||||
@@ -2401,6 +2407,102 @@ pub async fn fetch_reddit(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Streamable downloader (via yt-dlp)
|
||||
// ============================================================================
|
||||
|
||||
pub async fn fetch_streamable(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
if !url.contains("streamable.com") {
|
||||
return Err(ScrapingError::Http("Invalid Streamable URL".to_string()));
|
||||
}
|
||||
|
||||
let data = run_ytdlp_json(url, &["-f", "best"]).await?;
|
||||
|
||||
let title = data
|
||||
.get("title")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
let thumbnail = data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
|
||||
let mut result = DownloadResult::success(title);
|
||||
result.thumbnail = thumbnail;
|
||||
result.provider = Some("yt-dlp".to_string());
|
||||
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
if let Some(fmts) = data.get("formats").and_then(|v| v.as_array()) {
|
||||
for f in fmts {
|
||||
if let (Some(furl), Some(ext_val)) = (
|
||||
f.get("url").and_then(|v| v.as_str()),
|
||||
f.get("ext").and_then(|v| v.as_str()),
|
||||
) {
|
||||
if ext_val == "mhtml" {
|
||||
continue;
|
||||
}
|
||||
let url_string = furl.to_string();
|
||||
if seen.insert(url_string.clone()) {
|
||||
result.media.push(MediaItem {
|
||||
url: url_string,
|
||||
quality: f
|
||||
.get("height")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|h| format!("{}p", h)),
|
||||
file_type: Some(MediaType::Video),
|
||||
extension: Some(ext_val.to_string()),
|
||||
thumbnail: data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string()),
|
||||
file_size: f
|
||||
.get("filesize")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(format_filesize),
|
||||
size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
|
||||
frame_width: f
|
||||
.get("width")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|w| w.to_string()),
|
||||
frame_height: f
|
||||
.get("height")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|h| h.to_string()),
|
||||
note: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if result.media.is_empty() {
|
||||
if let Some(dl_url) = data.get("url").and_then(|v| v.as_str()) {
|
||||
result.media.push(MediaItem {
|
||||
url: dl_url.to_string(),
|
||||
quality: None,
|
||||
file_type: Some(MediaType::Video),
|
||||
extension: Some(
|
||||
data.get("ext")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("mp4")
|
||||
.to_string(),
|
||||
),
|
||||
thumbnail: data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string()),
|
||||
file_size: None,
|
||||
size_bytes: None,
|
||||
frame_width: None,
|
||||
frame_height: None,
|
||||
note: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Pinterest downloader
|
||||
// ============================================================================
|
||||
|
||||
@@ -211,6 +211,14 @@ pub async fn download_reddit(
|
||||
Ok(Json(DownloadResponse::ok(result)))
|
||||
}
|
||||
|
||||
/// Handler for _streamable.
|
||||
pub async fn download_streamable(
|
||||
Query(params): Query<DownloadParams>,
|
||||
) -> Result<Json<DownloadResponse>, AppError> {
|
||||
let result = use_cases::download_streamable(¶ms.url).await?;
|
||||
Ok(Json(DownloadResponse::ok(result)))
|
||||
}
|
||||
|
||||
/// Handler for _bilibili.
|
||||
pub async fn download_bilibili(
|
||||
Query(params): Query<DownloadParams>,
|
||||
|
||||
@@ -244,6 +244,10 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
||||
"/download/reddit",
|
||||
axum::routing::get(crate::presentation::handler::downloader::download_reddit),
|
||||
)
|
||||
.route(
|
||||
"/download/streamable",
|
||||
axum::routing::get(crate::presentation::handler::downloader::download_streamable),
|
||||
)
|
||||
.route(
|
||||
"/download/bilibili",
|
||||
axum::routing::get(crate::presentation::handler::downloader::download_bilibili),
|
||||
|
||||
Reference in New Issue
Block a user