feat: add Streamable downloader endpoint + fix detect false-positive
Deploy Scraper / build-and-deploy (push) Canceled after 0s
Deploy Scraper / build-and-deploy (push) Canceled after 0s
- /download/streamable: real MP4 via yt-dlp (verified ftypisom)
- Fix detect_platform: t.co substring greedily matched reddit.com
('.reddit.com' contains 't.co') - now requires t.co/ path
- Removed duplicate reddit branch in detect chain
This commit is contained in:
@@ -115,6 +115,10 @@ pub async fn download_reddit(url: &str) -> Result<DownloadResult, ScrapingError>
|
|||||||
DownloaderRepository::download_reddit(url).await
|
DownloaderRepository::download_reddit(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn download_streamable(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
DownloaderRepository::download_streamable(url).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
DownloaderRepository::download_bilibili(url).await
|
DownloaderRepository::download_bilibili(url).await
|
||||||
}
|
}
|
||||||
@@ -131,10 +135,12 @@ pub fn detect_platform(url: &str) -> String {
|
|||||||
"youtube".to_string()
|
"youtube".to_string()
|
||||||
} else if url.contains("open.spotify.com") || url.contains("spotify.link") {
|
} else if url.contains("open.spotify.com") || url.contains("spotify.link") {
|
||||||
"spotify".to_string()
|
"spotify".to_string()
|
||||||
} else if url.contains("twitter.com") || url.contains("x.com") || url.contains("t.co") {
|
} else if url.contains("twitter.com") || url.contains("x.com") || url.contains("t.co/") {
|
||||||
"twitter".to_string()
|
"twitter".to_string()
|
||||||
} else if url.contains("pinterest") {
|
} else if url.contains("pinterest") {
|
||||||
"pinterest".to_string()
|
"pinterest".to_string()
|
||||||
|
} else if url.contains("reddit.com") || url.contains("redd.it") {
|
||||||
|
"reddit".to_string()
|
||||||
} else if url.contains("mega.nz") || url.contains("mega.io") {
|
} else if url.contains("mega.nz") || url.contains("mega.io") {
|
||||||
"mega".to_string()
|
"mega".to_string()
|
||||||
} else if url.contains("terabox") || url.contains("nfile") {
|
} else if url.contains("terabox") || url.contains("nfile") {
|
||||||
@@ -157,8 +163,8 @@ pub fn detect_platform(url: &str) -> String {
|
|||||||
"soundcloud".to_string()
|
"soundcloud".to_string()
|
||||||
} else if url.contains("dailymotion.com") {
|
} else if url.contains("dailymotion.com") {
|
||||||
"dailymotion".to_string()
|
"dailymotion".to_string()
|
||||||
} else if url.contains("reddit.com") || url.contains("redd.it") {
|
} else if url.contains("streamable.com") {
|
||||||
"reddit".to_string()
|
"streamable".to_string()
|
||||||
} else if url.contains("bilibili.com") || url.contains("b23.tv") {
|
} else if url.contains("bilibili.com") || url.contains("b23.tv") {
|
||||||
"bilibili".to_string()
|
"bilibili".to_string()
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -470,6 +470,7 @@ impl DownloaderRepository {
|
|||||||
"soundcloud" => Self::download_soundcloud(url).await,
|
"soundcloud" => Self::download_soundcloud(url).await,
|
||||||
"dailymotion" => Self::download_dailymotion(url).await,
|
"dailymotion" => Self::download_dailymotion(url).await,
|
||||||
"reddit" => Self::download_reddit(url).await,
|
"reddit" => Self::download_reddit(url).await,
|
||||||
|
"streamable" => Self::download_streamable(url).await,
|
||||||
"bilibili" => Self::download_bilibili(url).await,
|
"bilibili" => Self::download_bilibili(url).await,
|
||||||
_ => fetch_all_in_one(url).await,
|
_ => fetch_all_in_one(url).await,
|
||||||
}
|
}
|
||||||
@@ -602,6 +603,11 @@ impl DownloaderRepository {
|
|||||||
fetch_reddit(url).await
|
fetch_reddit(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Streamable video via yt-dlp.
|
||||||
|
pub async fn download_streamable(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
fetch_streamable(url).await
|
||||||
|
}
|
||||||
|
|
||||||
/// Bilibili video via b23.tv short link expansion.
|
/// Bilibili video via b23.tv short link expansion.
|
||||||
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
fetch_bilibili(url).await
|
fetch_bilibili(url).await
|
||||||
@@ -2401,6 +2407,102 @@ pub async fn fetch_reddit(url: &str) -> Result<DownloadResult, ScrapingError> {
|
|||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Streamable downloader (via yt-dlp)
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
pub async fn fetch_streamable(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
if !url.contains("streamable.com") {
|
||||||
|
return Err(ScrapingError::Http("Invalid Streamable URL".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = run_ytdlp_json(url, &["-f", "best"]).await?;
|
||||||
|
|
||||||
|
let title = data
|
||||||
|
.get("title")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let thumbnail = data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
|
||||||
|
let mut result = DownloadResult::success(title);
|
||||||
|
result.thumbnail = thumbnail;
|
||||||
|
result.provider = Some("yt-dlp".to_string());
|
||||||
|
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
if let Some(fmts) = data.get("formats").and_then(|v| v.as_array()) {
|
||||||
|
for f in fmts {
|
||||||
|
if let (Some(furl), Some(ext_val)) = (
|
||||||
|
f.get("url").and_then(|v| v.as_str()),
|
||||||
|
f.get("ext").and_then(|v| v.as_str()),
|
||||||
|
) {
|
||||||
|
if ext_val == "mhtml" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let url_string = furl.to_string();
|
||||||
|
if seen.insert(url_string.clone()) {
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: url_string,
|
||||||
|
quality: f
|
||||||
|
.get("height")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|h| format!("{}p", h)),
|
||||||
|
file_type: Some(MediaType::Video),
|
||||||
|
extension: Some(ext_val.to_string()),
|
||||||
|
thumbnail: data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
|
file_size: f
|
||||||
|
.get("filesize")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(format_filesize),
|
||||||
|
size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
|
||||||
|
frame_width: f
|
||||||
|
.get("width")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|w| w.to_string()),
|
||||||
|
frame_height: f
|
||||||
|
.get("height")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|h| h.to_string()),
|
||||||
|
note: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.media.is_empty() {
|
||||||
|
if let Some(dl_url) = data.get("url").and_then(|v| v.as_str()) {
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: dl_url.to_string(),
|
||||||
|
quality: None,
|
||||||
|
file_type: Some(MediaType::Video),
|
||||||
|
extension: Some(
|
||||||
|
data.get("ext")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("mp4")
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
thumbnail: data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
|
file_size: None,
|
||||||
|
size_bytes: None,
|
||||||
|
frame_width: None,
|
||||||
|
frame_height: None,
|
||||||
|
note: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Pinterest downloader
|
// Pinterest downloader
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|||||||
@@ -211,6 +211,14 @@ pub async fn download_reddit(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Handler for _streamable.
|
||||||
|
pub async fn download_streamable(
|
||||||
|
Query(params): Query<DownloadParams>,
|
||||||
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
|
let result = use_cases::download_streamable(¶ms.url).await?;
|
||||||
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
|
}
|
||||||
|
|
||||||
/// Handler for _bilibili.
|
/// Handler for _bilibili.
|
||||||
pub async fn download_bilibili(
|
pub async fn download_bilibili(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
|
|||||||
@@ -244,6 +244,10 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
|||||||
"/download/reddit",
|
"/download/reddit",
|
||||||
axum::routing::get(crate::presentation::handler::downloader::download_reddit),
|
axum::routing::get(crate::presentation::handler::downloader::download_reddit),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/download/streamable",
|
||||||
|
axum::routing::get(crate::presentation::handler::downloader::download_streamable),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/download/bilibili",
|
"/download/bilibili",
|
||||||
axum::routing::get(crate::presentation::handler::downloader::download_bilibili),
|
axum::routing::get(crate::presentation::handler::downloader::download_bilibili),
|
||||||
|
|||||||
Reference in New Issue
Block a user