//! Media downloader repository — ports HTTP API results to domain types.
//!
//! Each method corresponds to a downloader from Shirokami-API.
//! All use the shared `http_client()` and proxy-fetch infrastructure.

use std::borrow::Cow;
use std::collections::HashMap;
use std::time::Duration;

use aes::cipher::{BlockDecrypt, KeyInit};
use base64::Engine;

use crate::domain::entity::downloader::{DownloadResult, MediaItem, MediaType};
use crate::domain::error::ScrapingError;
use crate::infrastructure::utils::http_client::http_client;
use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE, USER_AGENT};
use url::Url;

/// Default headers for outbound HTTP requests to external APIs.
#[allow(dead_code)]
fn api_headers() -> HeaderMap {
    let mut h = HeaderMap::new();
    h.insert(
        USER_AGENT,
        HeaderValue::from_static(
            "Mozilla/5.0 (Linux; Android 15; SM-F958 Build/AP3A.240905.015) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.6723.86 Mobile Safari/537.36",
        ),
    );
    h.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
    h
}

/// Helper: extract a YouTube video ID from a URL.
fn extract_youtube_id(url: &str) -> Option<String> {
    let patterns = [
        r"youtube\.com/watch\?v=([a-zA-Z0-9_-]{11})",
        r"youtu\.be/([a-zA-Z0-9_-]{11})",
        r"youtube\.com/shorts/([a-zA-Z0-9_-]{11})",
        r"youtube\.com/embed/([a-zA-Z0-9_-]{11})",
        r"youtube\.com/v/([a-zA-Z0-9_-]{11})",
    ];
    for pat in &patterns {
        if let Some(caps) = regex::Regex::new(pat).ok().and_then(|r| r.captures(url)) {
            if let Some(m) = caps.get(1) {
                return Some(m.as_str().to_string());
            }
        }
    }
    None
}

/// Helper: parse the video-id portion from TikTok/Douyin URL.
fn extract_tiktok_id(url: &str) -> Option<String> {
    if !url.contains("tiktok.com") && !url.contains("douyin.com") {
        return None;
    }
    // Handle short URLs like vm.tiktok.com/ZM8s5qJ6t — resolve redirect first
    if url.contains("vm.tiktok.com") || url.contains("vt.tiktok.com") {
        let re = regex::Regex::new(r"/([A-Za-z0-9_-]+)$").ok()?;
        let short_code = re.captures(url).and_then(|c| c.get(1))?.as_str().to_string();
        return Some(short_code);
    }
    // TikTok URLs contain an 18-20 digit video ID in the path
    let re = regex::Regex::new(r"/video/(\d{15,25})").ok()?;
    let caps = re.captures(url)?;
    Some(caps.get(1)?.as_str().to_string())
}

/// Locate the yt-dlp binary on the system.
/// Checks: PATH → /home/code/hermes-agent/.venv/bin/yt-dlp → common locations
fn find_ytdlp() -> Option<String> {
    // Check known locations first
    let candidates = [
        "/home/code/hermes-agent/.venv/bin/yt-dlp",
        "/usr/local/bin/yt-dlp",
        "/usr/bin/yt-dlp",
        "/snap/bin/yt-dlp",
        "/home/code/.local/bin/yt-dlp",
    ];
    for c in &candidates {
        if std::path::Path::new(c).exists() {
            return Some(c.to_string());
        }
    }
    // Check PATH
    let paths: Vec<_> = std::env::var_os("PATH")
        .into_iter()
        .flat_map(|p| std::env::split_paths(&p).collect::<Vec<_>>())
        .collect();
    for path in &paths {
        let candidate = path.join("yt-dlp");
        if candidate.exists() {
            return Some(candidate.to_string_lossy().into_owned());
        }
    }
    None
}

/// Run yt-dlp --dump-json and return the parsed JSON value.
/// Uses spawn + manual stdout reading to avoid pipe buffer truncation
/// on large outputs (>64KB on Linux default pipe buffer).
async fn run_ytdlp_json(url: &str, extra_args: &[&str]) -> Result<serde_json::Value, ScrapingError> {
    let ytdlp = find_ytdlp()
        .ok_or_else(|| ScrapingError::Http("yt-dlp binary not found".to_string()))?;

    let extra_args_owned: Vec<String> = extra_args.iter().map(|s| s.to_string()).collect();

    let (stdout_str, stderr_str, exit_code) = tokio::task::spawn_blocking({
        let url_owned = url.to_string();
        let ytdlp_owned = ytdlp.clone();
        move || {
            let mut cmd_args: Vec<String> = vec![
                "--dump-json".to_string(),
                "--no-warnings".to_string(),
                "--no-check-certificates".to_string(),
            ];
            cmd_args.extend(extra_args_owned.iter().cloned());
            cmd_args.push(url_owned);

            // Use Stdio::piped() + read_to_string to handle large stdout (64KB+).
            // Command::output() truncates at pipe buffer size.
            let mut child = std::process::Command::new(&ytdlp_owned)
                .args(&cmd_args)
                .stdout(std::process::Stdio::piped())
                .stderr(std::process::Stdio::piped())
                .spawn()
                .map_err(|e| ScrapingError::Http(format!("yt-dlp spawn failed: {}", e)))?;

            let stdout_file = child.stdout.take().unwrap();
            let stderr_file = child.stderr.take().unwrap();
            let mut stdout_str = String::new();
            let mut stderr_str = String::new();
            use std::io::Read;
            let mut stdout_handle = stdout_file;
            stdout_handle.read_to_string(&mut stdout_str).unwrap_or_default();
            let mut stderr_handle = stderr_file;
            stderr_handle.read_to_string(&mut stderr_str).unwrap_or_default();
            let status = child.wait().map_err(|e| ScrapingError::Http(format!("yt-dlp wait failed: {}", e)))?;

            Ok((stdout_str, stderr_str, status.code()))
        }
    })
    .await
    .map_err(|e| ScrapingError::Http(format!("yt-dlp execution failed: {}", e)))??;

    if exit_code != Some(0) {
        return Err(ScrapingError::Http(format!(
            "yt-dlp failed: {}",
            stderr_str.trim().lines().last().unwrap_or("unknown error")
        )));
    }

    // yt-dlp --dump-json outputs one JSON per line per format
    let json_line = stdout_str
        .lines()
        .next()
        .ok_or_else(|| ScrapingError::Http("yt-dlp produced no output".to_string()))?;

    serde_json::from_str(json_line)
        .map_err(|e| ScrapingError::Http(format!("yt-dlp JSON parse failed: {}", e)))
}

pub struct DownloaderRepository;

impl DownloaderRepository {
    pub fn new() -> Self {
        Self
    }

    /// Shared client getter — uses the global 30s-timeout client.
    #[allow(dead_code)]
    fn client(&self) -> &'static crate::infrastructure::utils::http_client::HttpClient {
        http_client()
    }

    /// All-in-one: auto-detect platform from URL and delegate to specialized
    /// downloader. Falls back to `downr.org` universal scraper.
    pub async fn download_all_in_one(
        url: &str,
        cookies: Option<&str>,
    ) -> Result<DownloadResult, ScrapingError> {
        let platform = crate::application::downloader::detect_platform(url);
        match platform.as_str() {
            "instagram" | "facebook" => match Self::download_instagram(url, cookies).await {
                ok @ Ok(_) => ok,
                Err(_) => Self::download_facebook(url, cookies).await,
            },
            "tiktok" => Self::download_tiktok(url, cookies).await,
            "youtube" => match Self::download_youtube(url, "720").await {
                ok @ Ok(_) => ok,
                Err(_) => Self::download_youtube_mp3(url).await,
            },
            "spotify" => Self::download_spotify(url, cookies).await,
            "twitter" => Self::download_twitter(url, cookies).await,
            "pinterest" => Self::download_pinterest(url).await,
            "mega" => Self::download_mega(url).await,
            "terabox" => Self::download_terabox(url).await,
            "gdrive" => Self::download_gdrive(url).await,
            "mediafire" => Self::download_mediafire(url).await,
            "pixeldrain" => Self::download_pixeldrain(url).await,
            "threads" => Self::download_threads(url, cookies).await,
            "doodstream" => Self::download_doodstream(url).await,
            "krakenfiles" => Self::download_krakenfiles(url).await,
            "danbooru" => Self::download_danbooru(url).await,
            "soundcloud" => Self::download_soundcloud(url).await,
            "bilibili" => Self::download_bilibili(url).await,
            _ => fetch_all_in_one(url).await,
        }
    }

    /// Instagram / Facebook via SnapSave.
    /// Cookies param accepted for API consistency but not required (SnapSave
    /// fetches its own).
    pub async fn download_instagram(
        url: &str,
        _cookies: Option<&str>,
    ) -> Result<DownloadResult, ScrapingError> {
        fetch_snapsave(url).await
    }

    /// Facebook via SnapSave.
    pub async fn download_facebook(
        url: &str,
        _cookies: Option<&str>,
    ) -> Result<DownloadResult, ScrapingError> {
        fetch_snapsave(url).await
    }

    /// TikTok via tikwm.com.
    pub async fn download_tiktok(
        url: &str,
        _cookies: Option<&str>,
    ) -> Result<DownloadResult, ScrapingError> {
        fetch_tiktok(url).await
    }

    /// YouTube video via savetube.media.
    pub async fn download_youtube(
        url: &str,
        quality: &str,
    ) -> Result<DownloadResult, ScrapingError> {
        fetch_youtube_mp4(url, quality).await
    }

    /// YouTube to MP3 via ydlp.yard.id.
    pub async fn download_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_youtube_mp3(url).await
    }

    /// Spotify via Spotify API (requires api_key).
    pub async fn download_spotify(
        url: &str,
        api_key: Option<&str>,
    ) -> Result<DownloadResult, ScrapingError> {
        let _key = match api_key {
            Some(k) => k.to_string(),
            None => std::env::var("SPOTIFY_API_KEY").unwrap_or_default(),
        };
        fetch_spotify(url).await
    }

    /// Twitter/X media via api.lrm.tube.
    pub async fn download_twitter(
        url: &str,
        _cookies: Option<&str>,
    ) -> Result<DownloadResult, ScrapingError> {
        fetch_twitter(url).await
    }

    /// Pinterest media via PinterestDownloader.
    pub async fn download_pinterest(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_pinterest(url).await
    }

    /// MEGA.nz file link resolution.
    pub async fn download_mega(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_mega(url).await
    }

    /// TeraBox / TeraFile direct link extraction.
    pub async fn download_terabox(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_terabox(url).await
    }

    /// Google Drive direct download link.
    pub async fn download_gdrive(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_gdrive(url).await
    }

    /// MediaFire direct download link.
    pub async fn download_mediafire(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_mediafire(url).await
    }

    /// PixelDrain file direct link.
    pub async fn download_pixeldrain(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_pixeldrain(url).await
    }

    /// Meta Threads media extraction.
    pub async fn download_threads(
        url: &str,
        _cookies: Option<&str>,
    ) -> Result<DownloadResult, ScrapingError> {
        fetch_threads(url, _cookies).await
    }

    /// DoodStream direct link extraction.
    pub async fn download_doodstream(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_doodstream(url).await
    }

    /// KrakenFiles direct download link.
    pub async fn download_krakenfiles(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_krakenfiles(url).await
    }

    /// Danbooru image post extraction.
    pub async fn download_danbooru(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_danbooru(url).await
    }

    /// SoundCloud track via ydlp converter.
    pub async fn download_soundcloud(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_soundcloud(url).await
    }

    /// Bilibili video via b23.tv short link expansion.
    pub async fn download_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
        fetch_bilibili(url).await
    }
}

// ============================================================================
// All-in-One downloader (downr.org)
// ============================================================================

pub async fn fetch_all_in_one(url: &str) -> Result<DownloadResult, ScrapingError> {
    let client = http_client();
    let headers = {
        let mut h = HeaderMap::new();
        h.insert("user-agent", HeaderValue::from_static(
            "Mozilla/5.0 (Linux; Android 15; SM-F958 Build/AP3A.240905.015) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.6723.86 Mobile Safari/537.36"
        ));
        h.insert("referer", HeaderValue::from_static("https://downr.org/"));
        h
    };

    // Step 1: get analytics to obtain cookies
    let analytics_resp = client
        .client()
        .get("https://downr.org/.netlify/functions/analytics")
        .headers(headers.clone())
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("Analytics fetch failed: {}", e)))?;

    let cookies: HashMap<String, String> = analytics_resp
        .headers()
        .get_all(reqwest::header::SET_COOKIE)
        .iter()
        .filter_map(|v| {
            let s = v.to_str().ok()?;
            let parts: Vec<&str> = s.split(';').next()?.splitn(2, '=').collect();
            if parts.len() == 2 {
                Some((parts[0].trim().to_string(), parts[1].trim().to_string()))
            } else {
                None
            }
        })
        .collect();

    let cookie_header = cookies
        .iter()
        .map(|(k, v)| format!("{}={}", k, v))
        .collect::<Vec<_>>()
        .join("; ");

    // Step 2: post download request
    let body = serde_json::json!({ "url": url });
    let mut req_headers = headers.clone();
    if !cookie_header.is_empty() {
        req_headers.insert(
            "cookie",
            HeaderValue::from_str(&cookie_header).unwrap_or_else(|_| HeaderValue::from_static("")),
        );
    }
    req_headers.insert("content-type", HeaderValue::from_static("application/json"));
    req_headers.insert("origin", HeaderValue::from_static("https://downr.org"));

    let resp = client
        .client()
        .post("https://downr.org/.netlify/functions/download")
        .headers(req_headers)
        .json(&body)
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("Download request failed: {}", e)))?;

    let data: serde_json::Value = resp
        .json()
        .await
        .map_err(|e| ScrapingError::Http(format!("JSON parse failed: {}", e)))?;

    let mut result = DownloadResult::success(
        data.get("title")
            .and_then(|v| v.as_str())
            .map(|s| s.to_string()),
    );
    result.author = data.get("author").and_then(|v| {
        v.get("name")
            .and_then(|n| n.as_str())
            .map(|s| s.to_string())
    });
    result.thumbnail = data
        .get("thumbnail")
        .and_then(|v| v.as_str())
        .map(|s| s.to_string());
    result.provider = Some("downr".to_string());

    if let Some(medias) = data.get("medias").and_then(|v| v.as_array()) {
        for m in medias {
            let item = MediaItem {
                url: m
                    .get("url")
                    .and_then(|v| v.as_str())
                    .unwrap_or("")
                    .to_string(),
                quality: m
                    .get("quality")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                file_type: m
                    .get("type")
                    .and_then(|v| v.as_str())
                    .and_then(|t| match t {
                        "video" => Some(MediaType::Video),
                        "audio" => Some(MediaType::Audio),
                        "image" => Some(MediaType::Image),
                        _ => Some(MediaType::File),
                    }),
                extension: m
                    .get("extension")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                thumbnail: m
                    .get("thumbnail")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                file_size: None,
                size_bytes: None,
                frame_width: None,
                frame_height: None,
                note: None,
            };
            result.media.push(item);
        }
    }

    Ok(result)
}

// ============================================================================
// Instagram downloader (snapsave.app)
// ============================================================================

/// Detect whether a URL points to an image or video, based on content-type,
/// magic bytes, or filename hints. Ports the logic from Shirokami's
/// `instagram.js` `detectType` function.
#[allow(dead_code)]
async fn detect_media_type(url: &str) -> MediaType {
    // thumb paths are images
    if regex::Regex::new(r"\/thumb(\?|$)")
        .ok()
        .map(|r| r.is_match(url))
        .unwrap_or(false)
    {
        return MediaType::Image;
    }

    // Try HEAD request for content-type
    let client = http_client();
    let ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
    if let Ok(resp) = client
        .client()
        .head(url)
        .header(USER_AGENT, ua)
        .timeout(Duration::from_secs(8))
        .send()
        .await
    {
        if let Some(ct) = resp
            .headers()
            .get(CONTENT_TYPE)
            .and_then(|h| h.to_str().ok())
        {
            let ct = ct.to_lowercase();
            if ct.starts_with("video/") {
                return MediaType::Video;
            }
            if ct.starts_with("image/") {
                return MediaType::Image;
            }
        }
    }

    // Magic bytes range GET (first 1KB)
    if let Ok(resp) = client
        .client()
        .get(url)
        .header("range", "bytes=0-1023")
        .header(USER_AGENT, ua)
        .header("accept", "*/*")
        .send()
        .await
    {
        if let Ok(bytes) = resp.bytes().await {
            if bytes.len() >= 12 {
                // JPEG: FF D8 FF
                if bytes[0] == 0xff && bytes[1] == 0xd8 && bytes[2] == 0xff {
                    return MediaType::Image;
                }
                // PNG
                if bytes[0..8] == [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] {
                    return MediaType::Image;
                }
                // WEBP: RIFF....WEBP
                if &bytes[0..4] == b"RIFF" && &bytes[8..12] == b"WEBP" {
                    return MediaType::Image;
                }
                // MP4: bytes 4..7 = 'ftyp'
                if &bytes[4..8] == b"ftyp" {
                    return MediaType::Video;
                }
            }
        }
    }

    // filename hints
    if let Ok(parsed) = Url::parse(url) {
        let query = parsed.query().unwrap_or("");
        for (k, v) in url::form_urlencoded::parse(query.as_bytes()) {
            if (k == "filename" || k == "file") && v.ends_with(".mp4") {
                return MediaType::Video;
            }
            if (k == "filename" || k == "file") && v.ends_with(".jpg") {
                return MediaType::Image;
            }
        }
    }

    MediaType::Image
}

/// SnapSave parser — extracts Instagram/Facebook media via snapsave.app.
/// Uses downr.org as fallback for robustness.
pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, ScrapingError> {
    // Validate Instagram/Facebook URL
    let valid_fb = regex::Regex::new(r"https?://(web\.|www\.|m\.)?(facebook|fb)\.(com|watch)\S+")
        .unwrap()
        .is_match(url);
    let valid_ig =
        regex::Regex::new(r"https?://(www\.)?instagram\.com/(p|reel|reels|tv|stories)/\S+")
            .unwrap()
            .is_match(url);

    if !valid_fb && !valid_ig {
        return Ok(DownloadResult::error(
            "Link Url not valid — only Instagram and Facebook URLs are supported",
        ));
    }

    // Delegate to universal downr.org scraper for robustness
    fetch_all_in_one(url).await
}
pub async fn fetch_tiktok(url: &str) -> Result<DownloadResult, ScrapingError> {
    if extract_tiktok_id(url).is_none() {
        return Ok(DownloadResult::error("Invalid URL"));
    }

    // Try tikwm API first; fall back to yt-dlp if blocked
    let resp = http_client()
        .client()
        .get("https://www.tikwm.com/api/")
        .query(&[("url", url), ("hd", "1")])
        .header(USER_AGENT, "Mozilla/5.0")
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("TikTok fetch failed: {}", e)))?;

    let resp_json: serde_json::Value = resp
        .json()
        .await
        .map_err(|e| ScrapingError::Http(format!("TikTok JSON parse failed: {}", e)))?;

    // Check if tikwm returned an error (e.g. Cloudflare blocked)
    let tikwm_code = resp_json.get("code");
    let tikwm_msg = resp_json.get("msg").and_then(|v| v.as_str());
    
    if tikwm_code == Some(&serde_json::Value::Number(serde_json::Number::from(0)))
        && tikwm_msg != Some("Url parsing is failed! Please check url.")
    {
        let mut result = DownloadResult::success(
            resp_json.get("data")
                .and_then(|d| d.get("title"))
                .and_then(|v| v.as_str())
                .map(|s| s.to_string()),
        );
        result.author = resp_json
            .get("data")
            .and_then(|d| d.get("author"))
            .and_then(|v| v.as_str())
            .map(|s| s.to_string());
        result.provider = Some("tikwm".to_string());
        result.thumbnail = resp_json
            .get("data")
            .and_then(|d| d.get("cover"))
            .and_then(|v| v.as_str())
            .map(|s| s.to_string());

        if let Some(plays) = resp_json.get("data").and_then(|d| d.get("plays").and_then(|v| v.as_array())) {
            for play in plays {
                if let Some(play_url) = play.get("url").and_then(|v| v.as_str()) {
                    result.media.push(MediaItem {
                        url: play_url.to_string(),
                        quality: play.get("quality").and_then(|v| v.as_str()).map(|s| s.to_string()),
                        file_type: Some(MediaType::Video),
                        extension: Some("mp4".to_string()),
                        thumbnail: None,
                        file_size: play.get("size").and_then(|v| v.as_str()).map(|s| s.to_string()),
                        size_bytes: play.get("size").and_then(|v| v.as_str()).and_then(|s| s.parse().ok()),
                        frame_width: None,
                        frame_height: None,
                        note: None,
                    });
                }
            }
        }

        if result.media.is_empty() {
            result.message = Some("No download URLs found".to_string());
            return Ok(result);
        }
        return Ok(result);
    }

    // Fallback: use yt-dlp
    let data = run_ytdlp_json(url, &[]).await?;
    let title = data.get("title").and_then(|v| v.as_str()).map(|s| s.to_string());
    let author = data.get("uploader").and_then(|v| v.as_str()).map(|s| s.to_string());
    let thumbnail = data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string());

    let mut result = DownloadResult::success(title);
    result.author = author;
    result.thumbnail = thumbnail;
    result.provider = Some("yt-dlp".to_string());

    if let Some(formats) = data.get("formats").and_then(|v| v.as_array()) {
        for fmt in formats {
            if let Some(fmt_url) = fmt.get("url").and_then(|v| v.as_str()) {
                result.media.push(MediaItem {
                    url: fmt_url.to_string(),
                    quality: fmt.get("format_note").and_then(|v| v.as_str()).map(|s| s.to_string())
                        .or_else(|| fmt.get("height").and_then(|v| v.as_str()).map(|s| s.to_string())),
                    file_type: fmt.get("vcodec").and_then(|v| v.as_str()).filter(|s| !s.is_empty() && *s != "none")
                        .map(|_| MediaType::Video),
                    extension: fmt.get("ext").and_then(|v| v.as_str()).map(|s| s.to_string()),
                    thumbnail: None,
                    file_size: fmt.get("filesize").and_then(|v| v.as_u64()).map(format_filesize),
                    size_bytes: fmt.get("filesize").and_then(|v| v.as_u64()),
                    frame_width: fmt.get("width").and_then(|v| v.as_str()).map(|s| s.to_string()),
                    frame_height: fmt.get("height").and_then(|v| v.as_str()).map(|s| s.to_string()),
                    note: None,
                });
            }
        }
    }

    Ok(result)
}


/// TikTok v2 — uses douyin.wtf API
pub async fn fetch_tiktok_v2(url: &str) -> Result<DownloadResult, ScrapingError> {
    let resp = http_client()
        .client()
        .get("https://douyin.wtf/api/hybrid/video_data")
        .query(&[("url", url), ("minimal", "true")])
        .header(USER_AGENT, "Mozilla/5.0")
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("TikTok v2 fetch failed: {}", e)))?
        .json::<serde_json::Value>()
        .await
        .map_err(|e| ScrapingError::Http(format!("TikTok v2 JSON parse failed: {}", e)))?;

    let data = &resp["data"];
    let mut result = DownloadResult::success(
        data.get("desc")
            .and_then(|v| v.as_str())
            .map(|s| s.to_string()),
    );
    result.author = data
        .get("author")
        .and_then(|v| v.as_str())
        .map(|s| s.to_string());
    result.provider = Some("douyin.wtf".to_string());

    if let Some(video_data) = data.get("video_data") {
        if let Some(url) = video_data.get("play").and_then(|v| v.as_str()) {
            result.media.push(MediaItem {
                url: url.to_string(),
                quality: video_data
                    .get("quality")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                file_type: Some(MediaType::Video),
                extension: Some("mp4".to_string()),
                thumbnail: video_data
                    .get("cover")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                file_size: video_data
                    .get("size")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                size_bytes: None,
                frame_width: None,
                frame_height: None,
                note: None,
            });
        }
        if let Some(url) = video_data.get("music").and_then(|v| v.as_str()) {
            result.media.push(MediaItem {
                url: url.to_string(),
                quality: Some("audio".to_string()),
                file_type: Some(MediaType::Audio),
                extension: Some("mp3".to_string()),
                thumbnail: None,
                file_size: None,
                size_bytes: None,
                frame_width: None,
                frame_height: None,
                note: None,
            });
        }
    }

    Ok(result)
}

// ============================================================================
// YouTube downloaders (via savetube.media)
// ============================================================================

/// YouTube audio via savetube.media — returns metadata + direct download link.
pub async fn fetch_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
    extract_youtube_id(url)
        .ok_or_else(|| ScrapingError::Http("Invalid YouTube URL".to_string()))?;

    let data = run_ytdlp_json(url, &["-f", "bestaudio/best"]).await?;

    let title = data.get("title").and_then(|v| v.as_str()).map(|s| s.to_string());
    let author = data.get("uploader").and_then(|v| v.as_str()).map(|s| s.to_string());
    let thumbnail = data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string());
    let duration = data.get("duration").and_then(|v| v.as_u64()).map(|d| format!("{}s", d));

    let mut result = DownloadResult::success(title);
    result.author = author;
    result.thumbnail = thumbnail;
    result.duration = duration;
    result.provider = Some("yt-dlp".to_string());

    let download_url = data.get("url").and_then(|v| v.as_str())
        .or_else(|| {
            data.get("formats")
                .and_then(|v| v.as_array())
                .and_then(|f| f.first())
                .and_then(|f| f.get("url").and_then(|v| v.as_str()))
        })
        .ok_or_else(|| ScrapingError::Http("No audio format URL from yt-dlp".to_string()))?;

    result.media.push(MediaItem {
        url: download_url.to_string(),
        quality: Some(format!("{}kbps", data.get("abr").and_then(|v| v.as_u64()).unwrap_or(128))),
        file_type: Some(MediaType::Audio),
        extension: Some("mp3".to_string()),
        thumbnail: data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string()),
        file_size: data.get("filesize").and_then(|v| v.as_u64()).map(format_filesize),
        size_bytes: data.get("filesize").and_then(|v| v.as_u64()),
        frame_width: None,
        frame_height: None,
        note: None,
    });

    Ok(result)
}

/// YouTube video via yt-dlp
pub async fn fetch_youtube_mp4(url: &str, quality: &str) -> Result<DownloadResult, ScrapingError> {
    extract_youtube_id(url)
        .ok_or_else(|| ScrapingError::Http("Invalid YouTube URL".to_string()))?;

    let fmt = format!("bestvideo[height<={}]/best+bestaudio/best", quality.trim_end_matches('p'));
    let data = run_ytdlp_json(url, &["-f", &fmt]).await?;

    let title = data.get("title").and_then(|v| v.as_str()).map(|s| s.to_string());
    let author = data.get("uploader").and_then(|v| v.as_str()).map(|s| s.to_string());
    let thumbnail = data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string());
    let duration = data.get("duration").and_then(|v| v.as_u64()).map(|d| format!("{}s", d));

    let mut result = DownloadResult::success(title);
    result.author = author;
    result.thumbnail = thumbnail;
    result.duration = duration;
    result.provider = Some("yt-dlp".to_string());

    let formats = data.get("formats").and_then(|v| v.as_array());
    let mut seen = std::collections::HashSet::new();

    if let Some(fmts) = formats {
        for f in fmts {
            if let (Some(furl), Some(ext_val)) = (
                f.get("url").and_then(|v| v.as_str()),
                f.get("ext").and_then(|v| v.as_str()),
            ) {
                if ext_val == "mhtml" { continue; }
                let url_string = furl.to_string();
                if seen.insert(url_string.clone()) {
                    let fmt_type = if ext_val == "mp4" || ext_val == "webm" || ext_val == "mkv" {
                        MediaType::Video
                    } else if ext_val == "mp3" || ext_val == "m4a" || ext_val == "webm" {
                        MediaType::Audio
                    } else {
                        MediaType::File
                    };
                    let q = format!("{}p", f.get("height").and_then(|v| v.as_u64()).unwrap_or(0));
                    result.media.push(MediaItem {
                        url: url_string,
                        quality: Some(q),
                        file_type: Some(fmt_type),
                        extension: Some(ext_val.to_string()),
                        thumbnail: data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string()),
                        file_size: f.get("filesize").and_then(|v| v.as_u64()).map(format_filesize),
                        size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
                        frame_width: f.get("width").and_then(|v| v.as_u64()).map(|w| w.to_string()),
                        frame_height: f.get("height").and_then(|v| v.as_u64()).map(|h| h.to_string()),
                        note: None,
                    });
                }
            }
        }
    }

    if result.media.is_empty() {
        if let Some(url) = data.get("url").and_then(|v| v.as_str()) {
            result.media.push(MediaItem {
                url: url.to_string(),
                quality: Some(format!("{}p", quality.trim_end_matches('p'))),
                file_type: Some(MediaType::Video),
                extension: Some(data.get("ext").and_then(|v| v.as_str()).unwrap_or("mp4").to_string()),
                thumbnail: data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string()),
                file_size: data.get("filesize").and_then(|v| v.as_u64()).map(format_filesize),
                size_bytes: data.get("filesize").and_then(|v| v.as_u64()),
                frame_width: None,
                frame_height: None,
                note: None,
            });
        }
    }

    Ok(result)
}

// Spotify downloaders
// ============================================================================

pub async fn fetch_spotify(url: &str) -> Result<DownloadResult, ScrapingError> {
    // Validate Spotify URL
    let re = regex::Regex::new(
        r"https?://open\.spotify\.com/(?:intl-[a-zA-Z0-9-]+/)?(track|album|playlist)/([a-zA-Z0-9]+)",
    )
    .unwrap();
    let captures = re
        .captures(url)
        .ok_or_else(|| ScrapingError::Http("Invalid Spotify URL".to_string()))?;
    let resource_type = captures.get(1).map(|m| m.as_str()).unwrap_or("track");
    let resource_id = captures.get(2).map(|m| m.as_str()).unwrap_or("");

    // Use yt-dlp to extract track info and download URL
    let data = run_ytdlp_json(url, &["--extract-audio", "--audio-format", "mp3"]).await?;

    let title = data.get("title").and_then(|v| v.as_str()).map(|s| s.to_string());
    let author = data.get("artist").and_then(|v| v.as_str()).map(|s| s.to_string());
    let thumbnail = data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string());
    let duration = data.get("duration").and_then(|v| v.as_u64()).map(|d| format!("{}s", d));

    let mut result = DownloadResult::success(title);
    result.author = author;
    result.thumbnail = thumbnail;
    result.duration = duration;
    result.provider = Some(format!("spotify-yt-dlp ({})", resource_type));

    let download_url = data.get("url").and_then(|v| v.as_str())
        .or_else(|| {
            data.get("formats")
                .and_then(|v| v.as_array())
                .and_then(|f| f.first())
                .and_then(|f| f.get("url").and_then(|v| v.as_str()))
        });

    if let Some(dl_url) = download_url {
        result.media.push(MediaItem {
            url: dl_url.to_string(),
            quality: Some("320kbps".to_string()),
            file_type: Some(MediaType::Audio),
            extension: Some("mp3".to_string()),
            thumbnail: data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string()),
            file_size: data.get("filesize").and_then(|v| v.as_u64()).map(format_filesize),
            size_bytes: data.get("filesize").and_then(|v| v.as_u64()),
            frame_width: None,
            frame_height: None,
            note: None,
        });
        Ok(result)
    } else {
        result.media.push(MediaItem {
            url: format!("https://open.spotify.com/track/{}", resource_id),
            quality: Some("metadata".to_string()),
            file_type: Some(MediaType::File),
            extension: Some("json".to_string()),
            thumbnail: data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string()),
            file_size: None,
            size_bytes: None,
            frame_width: None,
            frame_height: None,
            note: Some("Spotify track metadata extracted via yt-dlp. Direct audio download requires Spotify Premium.".to_string()),
        });
        Ok(result)
    }
}

// Twitter downloaders
// ============================================================================

pub async fn fetch_twitter(url: &str) -> Result<DownloadResult, ScrapingError> {
    let client = http_client();
    let ua = "PostmanRuntime/7.32.2";

    let resp = client
        .client()
        .post("https://savetwitter.net/api/ajaxSearch")
        .header(USER_AGENT, ua)
        .header("accept", "*/*")
        .header("content-type", "application/x-www-form-urlencoded")
        .form(&[("q", url), ("lang", "en")])
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("Twitter fetch failed: {}", e)))?
        .json::<serde_json::Value>()
        .await
        .map_err(|e| ScrapingError::Http(format!("Twitter JSON parse failed: {}", e)))?;

    let html = resp
        .get("data")
        .and_then(|v| v.as_str())
        .ok_or_else(|| {
            let msg = resp
                .get("msg")
                .and_then(|v| v.as_str())
                .unwrap_or("No data in Twitter response");
            ScrapingError::Http(msg.to_string())
        })?;

    let document = scraper::Html::parse_document(html);
    let mut result = DownloadResult::success(None);
    result.provider = Some("savetwitter".to_string());

    let tw_video_sel = scraper::Selector::parse("div.tw-video").unwrap();
    let _video_list_sel = scraper::Selector::parse("div.video-data > div > ul > li").unwrap();

    if document.select(&tw_video_sel).next().is_some() {
        if let Ok(item_sel) = scraper::Selector::parse("div.tw-right > div > p:nth-child(1) > a") {
            for item in document.select(&item_sel) {
                let quality_text = item.text().collect::<String>();
                let quality = if quality_text.contains("(") {
                    quality_text
                        .split("(")
                        .nth(1)
                        .and_then(|s| s.split("p").next())
                        .unwrap_or(&quality_text)
                        .trim()
                        .to_string()
                } else {
                    quality_text.trim().to_string()
                };
                let href = item.value().attr("href").unwrap_or("").to_string();
                result.media.push(MediaItem {
                    url: href,
                    quality: Some(quality),
                    file_type: Some(MediaType::Video),
                    extension: Some("mp4".to_string()),
                    thumbnail: None,
                    file_size: None,
                    size_bytes: None,
                    frame_width: None,
                    frame_height: None,
                    note: None,
                });
            }
        }
    } else {
        if let Ok(item_sel) = scraper::Selector::parse("div.video-data > div > ul > li") {
            for item in document.select(&item_sel) {
                let href = item
                    .select(&scraper::Selector::parse("div > div:nth-child(2) > a").unwrap())
                    .next()
                    .and_then(|a| a.value().attr("href"))
                    .map(|s| s.to_string())
                    .unwrap_or_default();
                if !href.is_empty() {
                    result.media.push(MediaItem {
                        url: href,
                        quality: None,
                        file_type: Some(MediaType::Image),
                        extension: Some("jpg".to_string()),
                        thumbnail: None,
                        file_size: None,
                        size_bytes: None,
                        frame_width: None,
                        frame_height: None,
                        note: None,
                    });
                }
            }
        }
    }

    if result.media.is_empty() {
        return Ok(DownloadResult::error("Tidak dapat menemukan video"));
    }

    // Sort by resolution desc
    result.media.sort_by(|a, b| {
        let qa = a
            .quality
            .as_ref()
            .and_then(|q| q.parse::<u32>().ok())
            .unwrap_or(0);
        let qb = b
            .quality
            .as_ref()
            .and_then(|q| q.parse::<u32>().ok())
            .unwrap_or(0);
        qb.cmp(&qa)
    });

    Ok(result)
}

/// Twitter v2 — uses twitsave.com
pub async fn fetch_twitter_v2(url: &str) -> Result<DownloadResult, ScrapingError> {
    let client = http_client();
    let html = client
        .client()
        .get(format!("https://twitsave.com/info?url={}", url))
        .header(USER_AGENT, "Mozilla/5.0")
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("Twitter v2 fetch failed: {}", e)))?
        .text()
        .await
        .map_err(|e| ScrapingError::Http(format!("Twitter v2 response read failed: {}", e)))?;

    let document = scraper::Html::parse_document(&html);
    let mut result = DownloadResult::success(None);
    result.provider = Some("twitsave".to_string());

    if let Ok(item_sel) = scraper::Selector::parse("div.origin-top-right > ul > li") {
        for item in document.select(&item_sel) {
            if let Some(a) = item.select(&scraper::Selector::parse("a").unwrap()).next() {
                let resolution_text = item
                    .select(&scraper::Selector::parse("div > div > div").unwrap())
                    .next()
                    .map(|d| d.text().collect::<String>())
                    .unwrap_or_default();
                if resolution_text.contains("Resolution: ") {
                    let parts: Vec<&str> = resolution_text
                        .trim_start_matches("Resolution: ")
                        .splitn(2, 'x')
                        .collect();
                    let width = parts.get(0).unwrap_or(&"").to_string();
                    let height = parts
                        .get(1)
                        .and_then(|s| s.parse::<u32>().ok())
                        .unwrap_or(0);
                    let video_url = a
                        .value()
                        .attr("href")
                        .map(|s| s.to_string())
                        .unwrap_or_default();
                    result.media.push(MediaItem {
                        url: video_url,
                        quality: Some(width),
                        file_type: Some(MediaType::Video),
                        extension: Some("mp4".to_string()),
                        thumbnail: None,
                        file_size: None,
                        size_bytes: None,
                        frame_width: None,
                        frame_height: Some(height.to_string()),
                        note: None,
                    });
                }
            }
        }
    }

    result.media.sort_by(|a, b| {
        let ha = a
            .frame_height
            .as_ref()
            .and_then(|h| h.parse::<u32>().ok())
            .unwrap_or(0);
        let hb = b
            .frame_height
            .as_ref()
            .and_then(|h| h.parse::<u32>().ok())
            .unwrap_or(0);
        hb.cmp(&ha)
    });

    if let Some(highest) = result.media.first().and_then(|m| m.frame_width.clone()) {
        result
            .media
            .retain(|m| m.frame_width.as_deref() == Some(&highest));
    }

    if result.media.is_empty() {
        return Ok(DownloadResult::error("Tidak dapat menemukan video"));
    }

    Ok(result)
}

// ============================================================================
// Bilibili downloader (cobalt-api)
// ============================================================================

pub async fn fetch_bilibili(url: &str) -> Result<DownloadResult, ScrapingError> {
    // yt-dlp supports both AV (/video/av123) and BV (/video/BV1xxx) IDs
    let data = run_ytdlp_json(url, &["-f", "bv*+ba/b"]).await?;

    let title = data.get("title").and_then(|v| v.as_str()).map(|s| s.to_string());
    let author = data.get("uploader").and_then(|v| v.as_str()).map(|s| s.to_string());
    let thumbnail = data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string());
    let duration = data.get("duration").and_then(|v| v.as_u64()).map(|d| format!("{}s", d));

    let mut result = DownloadResult::success(title);
    result.author = author;
    result.thumbnail = thumbnail;
    result.duration = duration;
    result.provider = Some("yt-dlp".to_string());

    // Extract media from formats array
    if let Some(formats) = data.get("formats").and_then(|v| v.as_array()) {
        for fmt in formats {
            let url = fmt.get("url").and_then(|v| v.as_str());
            if url.is_some() {
                result.media.push(MediaItem {
                    url: url.unwrap().to_string(),
                    quality: fmt.get("format_note").and_then(|v| v.as_str()).map(|s| s.to_string())
                        .or_else(|| fmt.get("height").and_then(|v| v.as_str()).map(|s| s.to_string())),
                    file_type: fmt.get("vcodec").and_then(|v| v.as_str()).filter(|s| !s.is_empty() && *s != "none")
                        .map(|_| MediaType::Video),
                    extension: fmt.get("ext").and_then(|v| v.as_str()).map(|s| s.to_string()),
                    thumbnail: None,
                    file_size: fmt.get("filesize").and_then(|v| v.as_u64()).map(format_filesize),
                    size_bytes: fmt.get("filesize").and_then(|v| v.as_u64()),
                    frame_width: fmt.get("width").and_then(|v| v.as_str()).map(|s| s.to_string()),
                    frame_height: fmt.get("height").and_then(|v| v.as_str()).map(|s| s.to_string()),
                    note: None,
                });
            }
        }
    }

    Ok(result)
}

// ============================================================================
// SoundCloud downloader
// ============================================================================

pub async fn fetch_soundcloud(url: &str) -> Result<DownloadResult, ScrapingError> {
    // yt-dlp --extract-audio requires a download; use --dump-json for direct URL
    let data = run_ytdlp_json(url, &[]).await?;

    let title = data.get("title").and_then(|v| v.as_str()).map(|s| s.to_string());
    let author = data.get("uploader").and_then(|v| v.as_str()).map(|s| s.to_string());
    let thumbnail = data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string());
    let duration = data.get("duration").and_then(|v| v.as_u64()).map(|d| format!("{}s", d));

    let mut result = DownloadResult::success(title);
    result.author = author;
    result.thumbnail = thumbnail;
    result.duration = duration;
    result.provider = Some("yt-dlp".to_string());

    let download_url = data.get("url").and_then(|v| v.as_str())
        .or_else(|| {
            data.get("formats")
                .and_then(|v| v.as_array())
                .and_then(|f| f.first())
                .and_then(|f| f.get("url").and_then(|v| v.as_str()))
        });

    if let Some(dl_url) = download_url {
        result.media.push(MediaItem {
            url: dl_url.to_string(),
            quality: Some(format!("{}kbps", data.get("abr").and_then(|v| v.as_u64()).unwrap_or(128))),
            file_type: Some(MediaType::Audio),
            extension: Some("mp3".to_string()),
            thumbnail: data.get("thumbnail").and_then(|v| v.as_str()).map(|s| s.to_string()),
            file_size: data.get("filesize").and_then(|v| v.as_u64()).map(format_filesize),
            size_bytes: data.get("filesize").and_then(|v| v.as_u64()),
            frame_width: None,
            frame_height: None,
            note: None,
        });
    }

    Ok(result)
}
// ============================================================================
// Pinterest downloader
// ============================================================================

pub async fn fetch_pinterest(url: &str) -> Result<DownloadResult, ScrapingError> {
    let client = http_client();
    let encoded = urlencoding::encode(url).to_string();

    let resp = client
        .client()
        .get(format!(
            "https://pinterestdownloader.io/frontendService/DownloaderService?url={}",
            encoded
        ))
        .header(USER_AGENT, "Mozilla/5.0")
        .header("accept", "*/*")
        .header("content-type", "application/json")
        .header("origin", "https://pinterestdownloader.io")
        .header("referer", "https://pinterestdownloader.io/")
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("Pinterest fetch failed: {}", e)))?
        .json::<serde_json::Value>()
        .await
        .map_err(|e| ScrapingError::Http(format!("Pinterest JSON parse failed: {}", e)))?;

    if !resp
        .get("success")
        .and_then(|v| v.as_bool())
        .unwrap_or(false)
    {
        return Ok(DownloadResult::error(
            resp.get("error")
                .and_then(|v| v.as_str())
                .unwrap_or("failed"),
        ));
    }

    let mut result = DownloadResult::success(None);
    result.provider = Some("pinterestdownloader".to_string());

    let originals: std::collections::HashSet<String> = std::collections::HashSet::new();
    let mut media_list: Vec<MediaItem> = Vec::new();

    if let Some(medias) = resp.get("media").and_then(|v| v.as_array()) {
        for m in medias {
            if m.get("extension").and_then(|v| v.as_str()) == Some("jpg")
                && m.get("url")
                    .and_then(|v| v.as_str())
                    .map_or(false, |u| u.contains("i.pinimg.com/"))
            {
                // Add original (high-res) variant
                if let Some(u) = m.get("url").and_then(|v| v.as_str()) {
                    let original_url = u.replace("/2/", "/originals/");
                    if !originals.contains(&original_url) {
                        media_list.push(MediaItem {
                            url: original_url.clone(),
                            quality: Some("original".to_string()),
                            file_type: Some(MediaType::Image),
                            extension: Some("jpg".to_string()),
                            thumbnail: m
                                .get("thumbnail")
                                .and_then(|v| v.as_str())
                                .map(|s| s.to_string()),
                            file_size: m
                                .get("size")
                                .and_then(|v| v.as_str())
                                .map(|s| s.to_string()),
                            size_bytes: None,
                            frame_width: None,
                            frame_height: None,
                            note: None,
                        });
                    }
                }
            }

            media_list.push(MediaItem {
                url: m
                    .get("url")
                    .and_then(|v| v.as_str())
                    .unwrap_or("")
                    .to_string(),
                quality: m
                    .get("quality")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                file_type: Some(MediaType::Image),
                extension: m
                    .get("extension")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                thumbnail: m
                    .get("thumbnail")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                file_size: m
                    .get("size")
                    .and_then(|v| v.as_str())
                    .map(|s| s.to_string()),
                size_bytes: None,
                frame_width: None,
                frame_height: None,
                note: None,
            });
        }
    }

    // Sort by size desc (like Shirokami)
    media_list.sort_by(|a, b| {
        let sa = a
            .file_size
            .as_ref()
            .and_then(|s| s.parse::<u64>().ok())
            .unwrap_or(0);
        let sb = b
            .file_size
            .as_ref()
            .and_then(|s| s.parse::<u64>().ok())
            .unwrap_or(0);
        sb.cmp(&sa)
    });

    result.media = media_list;
    Ok(result)
}

// ============================================================================
// Others: PixelDrain, DoodStream, TeraBox, Video, KrakenFiles, Mega, Danbooru
// ============================================================================

pub async fn fetch_pixeldrain(url: &str) -> Result<DownloadResult, ScrapingError> {
    let client = http_client();
    let html = client
        .client()
        .get(url)
        .header(USER_AGENT, "Mozilla/5.0")
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("PixelDrain fetch failed: {}", e)))?
        .text()
        .await
        .map_err(|e| ScrapingError::Http(format!("PixelDrain read failed: {}", e)))?;

    let re = regex::Regex::new(r"window\.viewer_data\s*=\s*(\{.*?\});").unwrap();
    let m = re
        .captures(&html)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str());
    let json_str =
        m.ok_or_else(|| ScrapingError::Http("Failed to retrieve viewer data".to_string()))?;

    let viewer_data: serde_json::Value = serde_json::from_str(json_str)
        .map_err(|e| ScrapingError::Http(format!("PixelDrain JSON parse failed: {}", e)))?;

    let file_data = &viewer_data["api_response"];
    let file_id = file_data["id"].as_str().unwrap_or("");

    let mut result = DownloadResult::success(file_data["name"].as_str().map(|s| s.to_string()));
    result.provider = Some("pixeldrain".to_string());

    result.media.push(MediaItem {
        url: format!("https://pixeldrain.com/api/file/{}?download", file_id),
        quality: None,
        file_type: Some(MediaType::File),
        extension: None,
        thumbnail: None,
        file_size: Some(format_filesize(file_data["size"].as_u64().unwrap_or(0))),
        size_bytes: file_data["size"].as_u64(),
        frame_width: None,
        frame_height: None,
        note: None,
    });

    Ok(result)
}

pub async fn fetch_doodstream(url: &str) -> Result<DownloadResult, ScrapingError> {
    let id = regex::Regex::new(r"/[de]/([a-zA-Z0-9]+)")
        .unwrap()
        .captures(url)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str())
        .ok_or_else(|| ScrapingError::Http("Linknya tidak bisa diproses".to_string()))?;

    let proxy = "https://rv.lil-hacker.workers.dev/proxy?mirror=dood&url=";
    let client = http_client();

    // Get metadata from dood.li page
    let page_html = client
        .client()
        .get(format!("https://dood.li/d/{}", id))
        .header(USER_AGENT, "Mozilla/5.0")
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("DoodStream page fetch failed: {}", e)))?
        .text()
        .await
        .map_err(|e| ScrapingError::Http(format!("DoodStream page read failed: {}", e)))?;

    // HTML parsing — extract all needed text BEFORE any .await to keep future Send
    // Scraper types (Html/HtmlElementRef/Selector) are NOT Send, so they must be
    // dropped before async points.
    let (page_title, page_length, page_uploadate) = {
        let document = scraper::Html::parse_document(&page_html);
        let text = |sel: &str| {
            document
                .select(&scraper::Selector::parse(sel).unwrap())
                .next()
                .map(|e| e.text().collect::<String>().trim().to_string())
                .unwrap_or_default()
        };
        (text(".title-wrap h4"), text(".length"), text(".uploadate"))
    };

    // Get embedded player page
    let ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
    let headers = {
        let mut h = HeaderMap::new();
        h.insert(USER_AGENT, HeaderValue::from_static(ua));
        h.insert("referer", HeaderValue::from_static("https://d000d.com/"));
        h
    };

    let embed_resp = client
        .client()
        .get(format!("{}{}/e/{}", proxy, "https://d000d.com", id))
        .headers(headers.clone())
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("DoodStream embed fetch failed: {}", e)))?
        .text()
        .await
        .map_err(|e| ScrapingError::Http(format!("DoodStream embed read failed: {}", e)))?;

    let cdn_match = regex::Regex::new(r"\$\.get\('([^']+)',")
        .unwrap()
        .captures(&embed_resp)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str().to_string());

    let cdn_path = cdn_match.ok_or_else(|| {
        ScrapingError::Http("Link Pass MD5 tidak ditemukan! Coba lagi nanti.".to_string())
    })?;

    // Generate random token (like JS crypto.randomBytes)
    let chars: String = (0..10)
        .map(|_| {
            let idx = fastrand::usize(..62);
            let c = if idx < 26 {
                (b'A' + idx as u8) as char
            } else if idx < 52 {
                (b'a' + (idx - 26) as u8) as char
            } else {
                (b'0' + (idx - 52) as u8) as char
            };
            c
        })
        .collect();

    let ds_resp = client
        .client()
        .get(format!("{}{}{}", proxy, "https://d000d.com", cdn_path))
        .headers(headers.clone())
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("DoodStream DS fetch failed: {}", e)))?
        .text()
        .await
        .map_err(|e| ScrapingError::Http(format!("DoodStream DS read failed: {}", e)))?;

    let _cm = cdn_path.split('/').last().unwrap_or("");

    let expiry = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .map(|d| d.as_millis())
        .unwrap_or(0);
    let direct_link = format!(
        "{}{}{}?token={}&expiry={}",
        proxy,
        "https://d000d.com",
        ds_resp.trim(),
        chars,
        expiry,
    );

    // Title/length/uploadate were extracted inside the scope block above
    // as owned strings to keep scraper::Html from crossing .await
    let (title, length_str, uploadate) = (page_title, page_length, page_uploadate);
    let mut result = DownloadResult::success(Some(title));
    result.provider = Some("doodstream".to_string());
    result.duration = Some(length_str);

    result.description = Some(uploadate);

    result.media.push(MediaItem {
        url: direct_link,
        quality: None,
        file_type: Some(MediaType::Video),
        extension: Some("mp4".to_string()),
        thumbnail: None,
        file_size: None,
        size_bytes: None,
        frame_width: None,
        frame_height: None,
        note: None,
    });

    Ok(result)
}

pub async fn fetch_terabox(url: &str) -> Result<DownloadResult, ScrapingError> {
    let pattern = regex::Regex::new(
        r"^https?://(?:www\.|1024)?terabox(?:app)?\.com/.*[?&]?surl=([a-zA-Z0-9_-]+)",
    )
    .unwrap();

    let mut surl = pattern
        .captures(url)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str().to_string());

    if surl.is_none() {
        // Try resolving redirect
        if !regex::Regex::new(r"^https?://(?:www\.|1024)?terabox(?:app)?\.com")
            .unwrap()
            .is_match(url)
        {
            return Ok(DownloadResult::error("Invalid TeraBox URL."));
        }

        let client = http_client();
        let resp = client
            .client()
            .get(url)
            .header(USER_AGENT, "Mozilla/5.0")
            .timeout(Duration::from_secs(15))
            .send()
            .await
            .map_err(|e| ScrapingError::Http(format!("TeraBox redirect fetch failed: {}", e)))?;

        let request_url = resp.url().to_string();
        surl = pattern
            .captures(&request_url)
            .and_then(|c| c.get(1))
            .map(|m| m.as_str().to_string());
    }

    let surl = surl.ok_or_else(|| ScrapingError::Http("SURL not found.".to_string()))?;

    let client = http_client();
    let resp = client
        .client()
        .get("https://tera2.sylyt93.workers.dev/info")
        .query(&[("s", &surl)])
        .header("origin", "https://www.kauruka.com")
        .header("referer", "https://www.kauruka.com/")
        .header(USER_AGENT, "Mozilla/5.0")
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("TeraBox info fetch failed: {}", e)))?
        .json::<serde_json::Value>()
        .await
        .map_err(|e| ScrapingError::Http(format!("TeraBox JSON parse failed: {}", e)))?;

    if let Some(download_url) = resp.get("url").and_then(|v| v.as_str()) {
        let mut result = DownloadResult::success(
            resp.get("name")
                .and_then(|v| v.as_str())
                .map(|s| s.to_string()),
        );
        result.provider = Some("terabox".to_string());
        result.thumbnail = resp
            .get("img")
            .and_then(|v| v.as_str())
            .map(|s| s.to_string());

        result.media.push(MediaItem {
            url: download_url.to_string(),
            quality: None,
            file_type: Some(MediaType::File),
            extension: None,
            thumbnail: None,
            file_size: None,
            size_bytes: None,
            frame_width: None,
            frame_height: None,
            note: None,
        });

        Ok(result)
    } else {
        Ok(DownloadResult::error(
            resp.get("error")
                .and_then(|v| v.as_str())
                .unwrap_or("Failed to fetch TeraBox info"),
        ))
    }
}

pub async fn fetch_videy(url: &str) -> Result<DownloadResult, ScrapingError> {
    let id = Url::parse(url)
        .map_err(|e| ScrapingError::Http(format!("Invalid URL: {}", e)))?
        .query_pairs()
        .find(|(k, _)| k == "id")
        .map(|(_, v)| v.into_owned())
        .ok_or_else(|| ScrapingError::Http("Invalid URL, missing \"id\" parameter".to_string()))?;

    let file_type = if id.len() == 9 && id.as_bytes().get(8) == Some(&b'2') {
        ".mov"
    } else {
        ".mp4"
    };

    let direct_url = format!("https://cdn.videy.co/{}{}", id, file_type);

    let mut result = DownloadResult::success(Some(format!("Videy video {}", id)));
    result.provider = Some("videy".to_string());

    result.media.push(MediaItem {
        url: direct_url,
        quality: None,
        file_type: Some(MediaType::Video),
        extension: Some(file_type.trim_start_matches('.').to_string()),
        thumbnail: None,
        file_size: None,
        size_bytes: None,
        frame_width: None,
        frame_height: None,
        note: None,
    });

    Ok(result)
}

pub async fn fetch_krakenfiles(url: &str) -> Result<DownloadResult, ScrapingError> {
    // Parse file ID from krakenfiles.com URL
    let file_id = regex::Regex::new(r"krakenfiles\.com/v/([A-Za-z0-9_-]+)")
        .unwrap()
        .captures(url)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str())
        .ok_or_else(|| ScrapingError::Http("Invalid KrakenFiles URL".to_string()))?;

    let client = http_client();
    let resp = client
        .client()
        .post(format!("https://krakenfiles.com/v/{}", file_id))
        .header(USER_AGENT, "Mozilla/5.0")
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("KrakenFiles fetch failed: {}", e)))?
        .json::<serde_json::Value>()
        .await
        .map_err(|e| ScrapingError::Http(format!("KrakenFiles JSON parse failed: {}", e)))?;

    let mut result = DownloadResult::success(
        resp.get("name")
            .and_then(|v| v.as_str())
            .map(|s| s.to_string()),
    );
    result.provider = Some("krakenfiles".to_string());

    result.media.push(MediaItem {
        url: resp
            .get("url")
            .and_then(|v| v.as_str())
            .unwrap_or("")
            .to_string(),
        quality: None,
        file_type: Some(MediaType::File),
        extension: resp
            .get("ext")
            .and_then(|v| v.as_str())
            .map(|s| s.to_string()),
        thumbnail: None,
        file_size: None,
        size_bytes: resp.get("size_b").and_then(|v| v.as_u64()),
        frame_width: None,
        frame_height: None,
        note: None,
    });

    Ok(result)
}

/// MEGA.nz — requires AES-128-CBC decryption of the file attributes.
/// Ported from Shirokami's `mega.js` crypto logic.
pub async fn fetch_mega(url: &str) -> Result<DownloadResult, ScrapingError> {
    let url_fixed = url.replace('#', "%23");
    let _parts: Vec<&str> = url_fixed.splitn(2, "#").collect();
    // Actually need to handle the # separator differently
    let cleaned = url.replace("#", "%23");
    let decoded_url = urlencoding::decode(&cleaned)
        .unwrap_or(Cow::Borrowed(&cleaned))
        .into_owned();

    let file_id = regex::Regex::new(r"file/([^#]+)")
        .unwrap()
        .captures(&decoded_url)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str().to_string());

    let file_id = file_id.ok_or_else(|| ScrapingError::Http("Not found".to_string()))?;

    let parts: Vec<&str> = decoded_url.splitn(2, '#').collect();
    let file_key = parts.get(1).copied().unwrap_or("");

    if file_key.is_empty() || file_key.len() != 43 {
        return Ok(DownloadResult::error(
            if file_key.is_empty() {
                "File key tidak ditemukan"
            } else if file_key.len() < 43 {
                "Not enough character"
            } else {
                "Too many character"
            }
            .to_string(),
        ));
    }

    let client = http_client();
    let headers = {
        let mut h = HeaderMap::new();
        h.insert(USER_AGENT, HeaderValue::from_static("Mozilla/5.0"));
        h.insert("origin", HeaderValue::from_static("https://mega.nz"));
        h.insert("referer", HeaderValue::from_static("https://mega.nz"));
        h
    };

    let payload = serde_json::json!([{ "a": "g", "g": 1, "p": file_id }]);

    // Retry loop (3 attempts)
    let mut data = None;
    for attempt in 0..3 {
        match client
            .client()
            .post("https://g.api.mega.co.nz/cs")
            .headers(headers.clone())
            .json(&payload)
            .timeout(Duration::from_secs(10))
            .send()
            .await
        {
            Ok(resp) => {
                data = Some(
                    resp.json::<Vec<serde_json::Value>>()
                        .await
                        .unwrap_or_default(),
                );
                break;
            }
            Err(_) if attempt < 2 => continue,
            Err(e) => {
                return Ok(DownloadResult::error(e.to_string()));
            }
        }
    }

    let data = data.unwrap_or_default();
    let first = data.first().unwrap_or(&serde_json::Value::Null);

    let mut attrs = None;
    if let Some(at) = first.get("at") {
        if let Some(dec) = decrypt_mega_attr(at.as_str().unwrap_or(""), file_key) {
            attrs = Some(dec);
        }
    }

    let mut result = DownloadResult::success(
        attrs
            .as_ref()
            .and_then(|a| a.get("n").and_then(|v| v.as_str()).map(|s| s.to_string())),
    );
    result.provider = Some("mega".to_string());
    result.title = attrs
        .as_ref()
        .and_then(|a| a.get("n").and_then(|v| v.as_str()).map(|s| s.to_string()));

    result.media.push(MediaItem {
        url: first
            .get("g")
            .and_then(|v| v.as_str())
            .unwrap_or("")
            .to_string(),
        quality: None,
        file_type: Some(MediaType::File),
        extension: None,
        thumbnail: None,
        file_size: first.get("s").and_then(|v| v.as_u64()).map(format_filesize),
        size_bytes: first.get("s").and_then(|v| v.as_u64()),
        frame_width: None,
        frame_height: None,
        note: None,
    });

    Ok(result)
}

/// MEGA attribute decryption — AES-128-CBC with the derived key.
fn decrypt_mega_attr(enc: &str, file_key: &str) -> Option<serde_json::Value> {
    use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64;

    let fixed_key = fix_base64_key(file_key);
    let key_buffer = BASE64.decode(&fixed_key).ok()?;
    if key_buffer.len() < 32 {
        return None;
    }

    // Derive 16-byte key from 32-byte file key
    let key_ints: Vec<u32> = (0..8)
        .map(|i| {
            u32::from_le_bytes([
                key_buffer[i * 4],
                key_buffer[i * 4 + 1],
                key_buffer[i * 4 + 2],
                key_buffer[i * 4 + 3],
            ])
        })
        .collect();

    let key_out: [u8; 16] = {
        let ints = [
            key_ints[0] ^ key_ints[4],
            key_ints[1] ^ key_ints[5],
            key_ints[2] ^ key_ints[6],
            key_ints[3] ^ key_ints[7],
        ];
        let mut buf = [0u8; 16];
        for (i, &val) in ints.iter().enumerate() {
            buf[i * 4..(i + 1) * 4].copy_from_slice(&val.to_le_bytes());
        }
        buf
    };

    let iv = [0u8; 16];
    decrypt_aes_128_cbc(&key_out, &iv, enc)
        .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
}

fn fix_base64_key(s: &str) -> String {
    let rem = s.len() % 4;
    if rem > 0 {
        format!("{}{}", s, "=".repeat(4 - rem))
    } else {
        s.to_string()
    }
}

fn decrypt_aes_128_cbc(key: &[u8], _iv: &[u8], enc: &str) -> Option<String> {
    use aes::Aes128;
    use base64::engine::general_purpose::URL_SAFE_NO_PAD as BASE64;

    let data = BASE64.decode(enc).ok()?;
    let cipher = Aes128::new_from_slice(key).ok()?;

    let mut result = Vec::new();
    for chunk in data.chunks_exact(16) {
        let mut block = [0u8; 16];
        block.copy_from_slice(chunk);
        if let Ok(_) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
            cipher.decrypt_block((&mut block).into())
        })) {
            result.extend_from_slice(&block);
        } else {
            return None;
        }
    }

    let s = String::from_utf8_lossy(&result).into_owned();
    let s = s.trim_start_matches("MEGA");
    Some(s.to_string())
}

/// Danbooru — returns direct image URL from post
pub async fn fetch_danbooru(url: &str) -> Result<DownloadResult, ScrapingError> {
    let post_id = regex::Regex::new(r"danbooru\.donmai\.us/posts/(\d+)$")
        .unwrap()
        .captures(url)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str())
        .ok_or_else(|| ScrapingError::Http("Invalid URL".to_string()))?;

    let client = http_client();
    let resp = client
        .client()
        .get(format!("https://danbooru.donmai.us/posts/{}.json", post_id))
        .header(USER_AGENT, "Mozilla/5.0")
        .timeout(Duration::from_secs(15))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("Danbooru fetch failed: {}", e)))?
        .json::<serde_json::Value>()
        .await
        .map_err(|e| ScrapingError::Http(format!("Danbooru JSON parse failed: {}", e)))?;

    let file_url = resp["file_url"]
        .as_str()
        .or_else(|| resp["large_file_url"].as_str())
        .ok_or_else(|| ScrapingError::Http("No file URL found".to_string()))?;

    let mut result = DownloadResult::success(resp["tag_string"].as_str().map(|s| s.to_string()));
    result.provider = Some("danbooru".to_string());

    let full_url = if file_url.starts_with("http") {
        file_url.to_string()
    } else {
        format!("https://danbooru.donmai.us{}", file_url)
    };

    let ext = full_url.rsplit('.').next().unwrap_or("jpg").to_string();
    let mut file_type = MediaType::Image;
    if ext == "mp4" || ext == "webm" || ext == "gif" {
        file_type = if ext == "gif" {
            MediaType::Image
        } else {
            MediaType::Video
        };
    }

    result.media.push(MediaItem {
        url: full_url,
        quality: None,
        file_type: Some(file_type),
        extension: Some(ext),
        thumbnail: resp["preview_file_url"].as_str().map(|s| s.to_string()),
        file_size: resp["file_size"].as_str().map(|s| s.to_string()),
        size_bytes: None,
        frame_width: resp["image_width"].as_str().map(|s| s.to_string()),
        frame_height: resp["image_height"].as_str().map(|s| s.to_string()),
        note: Some(resp["md5"].as_str().unwrap_or("").to_string()),
    });

    Ok(result)
}

// ============================================================================
// Helper functions
// ============================================================================

fn format_filesize(bytes: u64) -> String {
    const UNITS: &[&str] = &["bytes", "KB", "MB", "GB", "TB"];
    let mut size = bytes as f64;
    let mut unit = 0;
    while size >= 1024.0 && unit < UNITS.len() - 1 {
        size /= 1024.0;
        unit += 1;
    }
    format!("{:.2} {}", size, UNITS[unit])
}

// ============================================================================
// Google Drive direct download link — extracts the fileId from the share URL
// and builds a download URL via Google's uc handler.
// ============================================================================

pub async fn fetch_gdrive(url: &str) -> Result<DownloadResult, ScrapingError> {
    let mut result = DownloadResult::error("Invalid Google Drive URL");
    let id_re = regex::Regex::new(r"[?&]id=([a-zA-Z0-9_-]+)").unwrap();
    if let Some(caps) = id_re.captures(url) {
        let file_id = caps.get(1).unwrap().as_str();
        let download_url = format!("https://drive.google.com/uc?id={}&export=download", file_id);
        result = DownloadResult::success(None);
        result.provider = Some("gdrive".to_string());
        result.media.push(MediaItem {
            url: download_url,
            quality: None,
            file_type: Some(MediaType::File),
            extension: None,
            thumbnail: None,
            file_size: None,
            size_bytes: None,
            frame_width: None,
            frame_height: None,
            note: None,
        });
    }
    Ok(result)
}

// ============================================================================
// MediaFire direct download link — parses the MediaFire page for the
// direct download anchor.
// ============================================================================

pub async fn fetch_mediafire(url: &str) -> Result<DownloadResult, ScrapingError> {
    let client = http_client();
    let html = client
        .client()
        .get(url)
        .header(
            USER_AGENT,
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        )
        .header("accept", "text/html,application/xhtml+xml")
        .timeout(Duration::from_secs(20))
        .send()
        .await
        .map_err(|e| ScrapingError::Http(format!("MediaFire fetch failed: {}", e)))?
        .text()
        .await
        .map_err(|e| ScrapingError::Http(format!("MediaFire response read failed: {}", e)))?;

    let document = scraper::Html::parse_document(&html);
    let dl_sel = scraper::Selector::parse(r#"a#downloadButton"#).unwrap();
    let download_url = document
        .select(&dl_sel)
        .next()
        .and_then(|el| el.value().attr("href"))
        .map(|s| s.to_string());

    let title = {
        let title_sel = scraper::Selector::parse("h1").unwrap();
        document
            .select(&title_sel)
            .next()
            .map(|el| el.text().collect::<String>())
    };

    Ok(DownloadResult {
        title,
        status: crate::domain::entity::downloader::DownloadStatus::Success,
        media: download_url
            .map(|u| {
                vec![MediaItem {
                    url: u,
                    quality: None,
                    file_type: Some(MediaType::File),
                    extension: None,
                    thumbnail: None,
                    file_size: None,
                    size_bytes: None,
                    frame_width: None,
                    frame_height: None,
                    note: None,
                }]
            })
            .unwrap_or_default(),
        provider: Some("mediafire".to_string()),
        author: None,
        thumbnail: None,
        description: None,
        duration: None,
        message: None,
    })
}

// ============================================================================
// Threads media extraction — reuses the SnapSave flow (Instagram backend
// already handles Threads via the same snapsave.app endpoint).
// ============================================================================

pub async fn fetch_threads(
    url: &str,
    _cookies: Option<&str>,
) -> Result<DownloadResult, ScrapingError> {
    // Threads URLs go through SnapSave (same engine as Instagram)
    let mut result = fetch_snapsave(url).await?;
    if result.provider.is_none() {
        result.provider = Some("threads".to_string());
    }
    Ok(result)
}