feat: add Dailymotion + Reddit downloader endpoints via yt-dlp
Deploy Scraper / build-and-deploy (push) Canceled after 0s
Deploy Scraper / build-and-deploy (push) Canceled after 0s
New platforms from Shirokami-style downloadter set: - /download/dailymotion: real MP4/HLS streams (288p up to 2160p) - /download/reddit: real v.redd.it HLS media for public posts - detect + all-in-one dispatch updated Both verified working from VPS with real URLs.
This commit is contained in:
@@ -107,6 +107,14 @@ pub async fn download_soundcloud(url: &str) -> Result<DownloadResult, ScrapingEr
|
|||||||
DownloaderRepository::download_soundcloud(url).await
|
DownloaderRepository::download_soundcloud(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn download_dailymotion(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
DownloaderRepository::download_dailymotion(url).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn download_reddit(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
DownloaderRepository::download_reddit(url).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
DownloaderRepository::download_bilibili(url).await
|
DownloaderRepository::download_bilibili(url).await
|
||||||
}
|
}
|
||||||
@@ -147,6 +155,10 @@ pub fn detect_platform(url: &str) -> String {
|
|||||||
"danbooru".to_string()
|
"danbooru".to_string()
|
||||||
} else if url.contains("soundcloud.com") {
|
} else if url.contains("soundcloud.com") {
|
||||||
"soundcloud".to_string()
|
"soundcloud".to_string()
|
||||||
|
} else if url.contains("dailymotion.com") {
|
||||||
|
"dailymotion".to_string()
|
||||||
|
} else if url.contains("reddit.com") || url.contains("redd.it") {
|
||||||
|
"reddit".to_string()
|
||||||
} else if url.contains("bilibili.com") || url.contains("b23.tv") {
|
} else if url.contains("bilibili.com") || url.contains("b23.tv") {
|
||||||
"bilibili".to_string()
|
"bilibili".to_string()
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -468,6 +468,8 @@ impl DownloaderRepository {
|
|||||||
"krakenfiles" => Self::download_krakenfiles(url).await,
|
"krakenfiles" => Self::download_krakenfiles(url).await,
|
||||||
"danbooru" => Self::download_danbooru(url).await,
|
"danbooru" => Self::download_danbooru(url).await,
|
||||||
"soundcloud" => Self::download_soundcloud(url).await,
|
"soundcloud" => Self::download_soundcloud(url).await,
|
||||||
|
"dailymotion" => Self::download_dailymotion(url).await,
|
||||||
|
"reddit" => Self::download_reddit(url).await,
|
||||||
"bilibili" => Self::download_bilibili(url).await,
|
"bilibili" => Self::download_bilibili(url).await,
|
||||||
_ => fetch_all_in_one(url).await,
|
_ => fetch_all_in_one(url).await,
|
||||||
}
|
}
|
||||||
@@ -590,6 +592,16 @@ impl DownloaderRepository {
|
|||||||
fetch_soundcloud(url).await
|
fetch_soundcloud(url).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Dailymotion video via yt-dlp.
|
||||||
|
pub async fn download_dailymotion(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
fetch_dailymotion(url).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reddit video via yt-dlp.
|
||||||
|
pub async fn download_reddit(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
fetch_reddit(url).await
|
||||||
|
}
|
||||||
|
|
||||||
/// Bilibili video via b23.tv short link expansion.
|
/// Bilibili video via b23.tv short link expansion.
|
||||||
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
fetch_bilibili(url).await
|
fetch_bilibili(url).await
|
||||||
@@ -2155,6 +2167,240 @@ pub async fn fetch_soundcloud(url: &str) -> Result<DownloadResult, ScrapingError
|
|||||||
|
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
// ============================================================================
|
||||||
|
// Dailymotion downloader (via yt-dlp)
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
pub async fn fetch_dailymotion(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
if !url.contains("dailymotion.com") {
|
||||||
|
return Err(ScrapingError::Http("Invalid Dailymotion URL".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = run_ytdlp_json(url, &["-f", "bestvideo+bestaudio/best"]).await?;
|
||||||
|
|
||||||
|
let title = data
|
||||||
|
.get("title")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let author = data
|
||||||
|
.get("uploader")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let thumbnail = data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let duration = data
|
||||||
|
.get("duration")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|d| format!("{}s", d));
|
||||||
|
|
||||||
|
let mut result = DownloadResult::success(title);
|
||||||
|
result.author = author;
|
||||||
|
result.thumbnail = thumbnail;
|
||||||
|
result.duration = duration;
|
||||||
|
result.provider = Some("yt-dlp".to_string());
|
||||||
|
|
||||||
|
let formats = data.get("formats").and_then(|v| v.as_array());
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
|
||||||
|
if let Some(fmts) = formats {
|
||||||
|
for f in fmts {
|
||||||
|
if let (Some(furl), Some(ext_val)) = (
|
||||||
|
f.get("url").and_then(|v| v.as_str()),
|
||||||
|
f.get("ext").and_then(|v| v.as_str()),
|
||||||
|
) {
|
||||||
|
if ext_val == "mhtml" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let url_string = furl.to_string();
|
||||||
|
if seen.insert(url_string.clone()) {
|
||||||
|
let fmt_type = if ext_val == "mp4" || ext_val == "webm" || ext_val == "mkv" {
|
||||||
|
MediaType::Video
|
||||||
|
} else if ext_val == "mp3" || ext_val == "m4a" {
|
||||||
|
MediaType::Audio
|
||||||
|
} else {
|
||||||
|
MediaType::File
|
||||||
|
};
|
||||||
|
let q = format!("{}p", f.get("height").and_then(|v| v.as_u64()).unwrap_or(0));
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: url_string,
|
||||||
|
quality: Some(q),
|
||||||
|
file_type: Some(fmt_type),
|
||||||
|
extension: Some(ext_val.to_string()),
|
||||||
|
thumbnail: data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
|
file_size: f
|
||||||
|
.get("filesize")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(format_filesize),
|
||||||
|
size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
|
||||||
|
frame_width: f
|
||||||
|
.get("width")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|w| w.to_string()),
|
||||||
|
frame_height: f
|
||||||
|
.get("height")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|h| h.to_string()),
|
||||||
|
note: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no formats listed, fall back to the top-level url
|
||||||
|
if result.media.is_empty() {
|
||||||
|
if let Some(dl_url) = data.get("url").and_then(|v| v.as_str()) {
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: dl_url.to_string(),
|
||||||
|
quality: Some(format!(
|
||||||
|
"{}p",
|
||||||
|
data.get("height").and_then(|v| v.as_u64()).unwrap_or(0)
|
||||||
|
)),
|
||||||
|
file_type: Some(MediaType::Video),
|
||||||
|
extension: Some(
|
||||||
|
data.get("ext")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("mp4")
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
thumbnail: data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
|
file_size: None,
|
||||||
|
size_bytes: None,
|
||||||
|
frame_width: None,
|
||||||
|
frame_height: None,
|
||||||
|
note: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Reddit downloader (via yt-dlp)
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
pub async fn fetch_reddit(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
if !url.contains("reddit.com") && !url.contains("redd.it") {
|
||||||
|
return Err(ScrapingError::Http("Invalid Reddit URL".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = run_ytdlp_json(url, &["-f", "bestvideo+bestaudio/best"]).await?;
|
||||||
|
|
||||||
|
let title = data
|
||||||
|
.get("title")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let author = data
|
||||||
|
.get("uploader")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let thumbnail = data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let duration = data
|
||||||
|
.get("duration")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|d| format!("{}s", d));
|
||||||
|
|
||||||
|
let mut result = DownloadResult::success(title);
|
||||||
|
result.author = author;
|
||||||
|
result.thumbnail = thumbnail;
|
||||||
|
result.duration = duration;
|
||||||
|
result.provider = Some("yt-dlp".to_string());
|
||||||
|
|
||||||
|
let formats = data.get("formats").and_then(|v| v.as_array());
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
|
||||||
|
if let Some(fmts) = formats {
|
||||||
|
for f in fmts {
|
||||||
|
if let (Some(furl), Some(ext_val)) = (
|
||||||
|
f.get("url").and_then(|v| v.as_str()),
|
||||||
|
f.get("ext").and_then(|v| v.as_str()),
|
||||||
|
) {
|
||||||
|
if ext_val == "mhtml" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let url_string = furl.to_string();
|
||||||
|
if seen.insert(url_string.clone()) {
|
||||||
|
let fmt_type = if ext_val == "mp4" || ext_val == "webm" || ext_val == "mkv" {
|
||||||
|
MediaType::Video
|
||||||
|
} else if ext_val == "mp3" || ext_val == "m4a" {
|
||||||
|
MediaType::Audio
|
||||||
|
} else {
|
||||||
|
MediaType::File
|
||||||
|
};
|
||||||
|
let q = format!("{}p", f.get("height").and_then(|v| v.as_u64()).unwrap_or(0));
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: url_string,
|
||||||
|
quality: Some(q),
|
||||||
|
file_type: Some(fmt_type),
|
||||||
|
extension: Some(ext_val.to_string()),
|
||||||
|
thumbnail: data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
|
file_size: f
|
||||||
|
.get("filesize")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(format_filesize),
|
||||||
|
size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
|
||||||
|
frame_width: f
|
||||||
|
.get("width")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|w| w.to_string()),
|
||||||
|
frame_height: f
|
||||||
|
.get("height")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|h| h.to_string()),
|
||||||
|
note: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no formats listed, fall back to the top-level url
|
||||||
|
if result.media.is_empty() {
|
||||||
|
if let Some(dl_url) = data.get("url").and_then(|v| v.as_str()) {
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: dl_url.to_string(),
|
||||||
|
quality: Some(format!(
|
||||||
|
"{}p",
|
||||||
|
data.get("height").and_then(|v| v.as_u64()).unwrap_or(0)
|
||||||
|
)),
|
||||||
|
file_type: Some(MediaType::Video),
|
||||||
|
extension: Some(
|
||||||
|
data.get("ext")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("mp4")
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
thumbnail: data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
|
file_size: None,
|
||||||
|
size_bytes: None,
|
||||||
|
frame_width: None,
|
||||||
|
frame_height: None,
|
||||||
|
note: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Pinterest downloader
|
// Pinterest downloader
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|||||||
@@ -195,6 +195,22 @@ pub async fn download_soundcloud(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Handler for _dailymotion.
|
||||||
|
pub async fn download_dailymotion(
|
||||||
|
Query(params): Query<DownloadParams>,
|
||||||
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
|
let result = use_cases::download_dailymotion(¶ms.url).await?;
|
||||||
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Handler for _reddit.
|
||||||
|
pub async fn download_reddit(
|
||||||
|
Query(params): Query<DownloadParams>,
|
||||||
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
|
let result = use_cases::download_reddit(¶ms.url).await?;
|
||||||
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
|
}
|
||||||
|
|
||||||
/// Handler for _bilibili.
|
/// Handler for _bilibili.
|
||||||
pub async fn download_bilibili(
|
pub async fn download_bilibili(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
|
|||||||
@@ -236,6 +236,14 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
|||||||
"/download/soundcloud",
|
"/download/soundcloud",
|
||||||
axum::routing::get(crate::presentation::handler::downloader::download_soundcloud),
|
axum::routing::get(crate::presentation::handler::downloader::download_soundcloud),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/download/dailymotion",
|
||||||
|
axum::routing::get(crate::presentation::handler::downloader::download_dailymotion),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/download/reddit",
|
||||||
|
axum::routing::get(crate::presentation::handler::downloader::download_reddit),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/download/bilibili",
|
"/download/bilibili",
|
||||||
axum::routing::get(crate::presentation::handler::downloader::download_bilibili),
|
axum::routing::get(crate::presentation::handler::downloader::download_bilibili),
|
||||||
|
|||||||
Reference in New Issue
Block a user