MythEclipse/scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Pinterest Playwright fallback + Spotify graceful DRM message
Deploy Scraper / build-and-deploy (push) Canceled after 0s
Details

Browse Source 
- fetch_pinterest: pinterestdownloader.io API failures now non-fatal, fall
  through to Playwright scraping (captures v.pinimg.com video URLs); returns
  graceful 200 'No download URLs found' instead of 502 on dead API.
- fetch_spotify: catch yt-dlp DRM error and return 200 with track metadata +
  clear message (direct audio needs Spotify Premium). No more raw 502.
This commit is contained in:
asepharyana
2026-09-01 16:36:36 +07:00
parent 563bce5211
commit 6b3fad1ad4
1 changed files with 139 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/infrastructure/repository/downloader.rs
+139 -96
View File
@@ -1393,8 +1393,34 @@ pub async fn fetch_spotify(url: &str) -> Result<DownloadResult, ScrapingError> {
let resource_type = captures.get(1).map(|m| m.as_str()).unwrap_or("track"); let resource_type = captures.get(1).map(|m| m.as_str()).unwrap_or("track");
let resource_id = captures.get(2).map(|m| m.as_str()).unwrap_or(""); let resource_id = captures.get(2).map(|m| m.as_str()).unwrap_or("");
// Use yt-dlp to extract track info and download URL // Use yt-dlp to extract track info and download URL.
let data = run_ytdlp_json(url, &["--extract-audio", "--audio-format", "mp3"]).await?; // Spotify tracks are DRM-protected: yt-dlp returns a "DRM" error without
// Premium auth, so catch that and return a clean informational message.
let data = match run_ytdlp_json(url, &["--extract-audio", "--audio-format", "mp3"]).await {
Ok(d) => d,
Err(_) => {
// Spotify tracks are DRM-protected (yt-dlp returns a "DRM" error without
// Premium auth). Return a clean informational message instead of an error.
let mut result = DownloadResult::success(None);
result.provider = Some(format!("spotify-metadata ({})", resource_type));
result.media.push(MediaItem {
url: format!("https://open.spotify.com/{}/{}", resource_type, resource_id),
quality: Some("metadata".to_string()),
file_type: Some(MediaType::File),
extension: Some("json".to_string()),
thumbnail: None,
file_size: None,
size_bytes: None,
frame_width: None,
frame_height: None,
note: Some(
"Spotify is DRM-protected: direct audio download requires a Spotify Premium account. Returned track metadata instead."
.to_string(),
),
});
return Ok(result);
}
};
let title = data let title = data
.get("title") .get("title")
@@ -1953,7 +1979,9 @@ pub async fn fetch_pinterest(url: &str) -> Result<DownloadResult, ScrapingError>
let client = http_client(); let client = http_client();
let encoded = urlencoding::encode(url).to_string(); let encoded = urlencoding::encode(url).to_string();
let resp = client // Try the pinterestdownloader.io API — but treat any failure (network, non-JSON,
// error body) as non-fatal so we can fall through to Playwright scraping below.
let resp_opt = match client
.client() .client()
.get(format!( .get(format!(
"https://pinterestdownloader.io/frontendService/DownloaderService?url={}", "https://pinterestdownloader.io/frontendService/DownloaderService?url={}",
@@ -1967,109 +1995,124 @@ pub async fn fetch_pinterest(url: &str) -> Result<DownloadResult, ScrapingError>
.timeout(Duration::from_secs(15)) .timeout(Duration::from_secs(15))
.send() .send()
.await .await
.map_err(|e| ScrapingError::Http(format!("Pinterest fetch failed: {}", e)))?
.json::<serde_json::Value>()
.await
.map_err(|e| ScrapingError::Http(format!("Pinterest JSON parse failed: {}", e)))?;
if !resp
.get("success")
.and_then(|v| v.as_bool())
.unwrap_or(false)
{ {
return Ok(DownloadResult::error( Ok(r) => match r.text().await {
resp.get("error") Ok(body) => serde_json::from_str::<serde_json::Value>(&body)
.and_then(|v| v.as_str()) .ok()
.unwrap_or("failed"), .filter(|resp| {
)); resp.get("success")
} .and_then(|v| v.as_bool())
.unwrap_or(false)
}),
Err(_) => None,
},
Err(_) => None,
};
let mut result = DownloadResult::success(None); let mut result = DownloadResult::success(None);
result.provider = Some("pinterestdownloader".to_string());
let originals: std::collections::HashSet<String> = std::collections::HashSet::new(); if let Some(resp) = resp_opt {
let mut media_list: Vec<MediaItem> = Vec::new(); result.provider = Some("pinterestdownloader".to_string());
if let Some(medias) = resp.get("media").and_then(|v| v.as_array()) { let originals: std::collections::HashSet<String> = std::collections::HashSet::new();
for m in medias { let mut media_list: Vec<MediaItem> = Vec::new();
if m.get("extension").and_then(|v| v.as_str()) == Some("jpg")
&& m.get("url") if let Some(medias) = resp.get("media").and_then(|v| v.as_array()) {
.and_then(|v| v.as_str()) for m in medias {
.map_or(false, |u| u.contains("i.pinimg.com/")) if m.get("extension").and_then(|v| v.as_str()) == Some("jpg")
{ && m.get("url")
// Add original (high-res) variant .and_then(|v| v.as_str())
if let Some(u) = m.get("url").and_then(|v| v.as_str()) { .map_or(false, |u| u.contains("i.pinimg.com/"))
let original_url = u.replace("/2/", "/originals/"); {
if !originals.contains(&original_url) { // Add original (high-res) variant
media_list.push(MediaItem { if let Some(u) = m.get("url").and_then(|v| v.as_str()) {
url: original_url.clone(), let original_url = u.replace("/2/", "/originals/");
quality: Some("original".to_string()), if !originals.contains(&original_url) {
file_type: Some(MediaType::Image), media_list.push(MediaItem {
extension: Some("jpg".to_string()), url: original_url.clone(),
thumbnail: m quality: Some("original".to_string()),
.get("thumbnail") file_type: Some(MediaType::Image),
.and_then(|v| v.as_str()) extension: Some("jpg".to_string()),
.map(|s| s.to_string()), thumbnail: m
file_size: m .get("thumbnail")
.get("size") .and_then(|v| v.as_str())
.and_then(|v| v.as_str()) .map(|s| s.to_string()),
.map(|s| s.to_string()), file_size: m
size_bytes: None, .get("size")
frame_width: None, .and_then(|v| v.as_str())
frame_height: None, .map(|s| s.to_string()),
note: None, size_bytes: None,
}); frame_width: None,
frame_height: None,
note: None,
});
}
} }
} }
}
media_list.push(MediaItem { media_list.push(MediaItem {
url: m url: m
.get("url") .get("url")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.unwrap_or("") .unwrap_or("")
.to_string(), .to_string(),
quality: m quality: m
.get("quality") .get("quality")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.map(|s| s.to_string()), .map(|s| s.to_string()),
file_type: Some(MediaType::Image), file_type: Some(MediaType::Image),
extension: m extension: m
.get("extension") .get("extension")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.map(|s| s.to_string()), .map(|s| s.to_string()),
thumbnail: m thumbnail: m
.get("thumbnail") .get("thumbnail")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.map(|s| s.to_string()), .map(|s| s.to_string()),
file_size: m file_size: m
.get("size") .get("size")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.map(|s| s.to_string()), .map(|s| s.to_string()),
size_bytes: None, size_bytes: None,
frame_width: None, frame_width: None,
frame_height: None, frame_height: None,
note: None, note: None,
}); });
}
}
// Sort by size desc (like Shirokami)
media_list.sort_by(|a, b| {
let sa = a
.file_size
.as_ref()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
let sb = b
.file_size
.as_ref()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
sb.cmp(&sa)
});
result.media = media_list;
} // end if let Some(resp) — API returned nothing on error, fall through below
// If the pinterestdownloader.io API returned nothing, fall back to
// Playwright browser scraping (same pattern as Instagram/Facebook) —
// the Playwright scraper captures v.pinimg.com video URLs from network responses.
if result.media.is_empty() {
if let Ok(data) = run_playwright_scraper(url, "pinterest").await {
let pw_result = playwright_to_download_result(&data);
if !pw_result.media.is_empty() {
return Ok(pw_result);
}
} }
} }
if result.media.is_empty() {
// Sort by size desc (like Shirokami) result.message = Some("No download URLs found".to_string());
media_list.sort_by(|a, b| { }
let sa = a
.file_size
.as_ref()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
let sb = b
.file_size
.as_ref()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
sb.cmp(&sa)
});
result.media = media_list;
Ok(result) Ok(result)
} }