Compare commits
9
Commits
bbb8bed389
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7d68ff5620 | ||
|
|
8b9bdeb56a | ||
|
|
029837d0f1 | ||
|
|
8a55fd91a4 | ||
|
|
5cec284cd6 | ||
|
|
adfbbfcea7 | ||
|
|
09c706cc61 | ||
|
|
93828c7253 | ||
|
|
5470d74921 |
No files matched your search
@@ -15,7 +15,7 @@
|
||||
};
|
||||
|
||||
scraper = pkgs.stdenv.mkDerivation {
|
||||
name = "scraper-0.1.0";
|
||||
name = "scraper-0.1.1";
|
||||
src = ./.;
|
||||
|
||||
nativeBuildInputs = with pkgs; [
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
//! Application use-cases for AI endpoints (Pollinations).
|
||||
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::infrastructure::repository::ai as Repo;
|
||||
use serde_json::Value;
|
||||
|
||||
pub async fn chat(
|
||||
text: String,
|
||||
prompt: String,
|
||||
model: Option<String>,
|
||||
) -> Result<Value, ScrapingError> {
|
||||
Repo::chat(text, prompt, model)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
pub async fn gemini(
|
||||
text: String,
|
||||
prompt: String,
|
||||
image_url: Option<String>,
|
||||
) -> Result<Value, ScrapingError> {
|
||||
Repo::gemini(text, prompt, image_url)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
pub async fn image(
|
||||
prompt: String,
|
||||
model: String,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<Vec<u8>, ScrapingError> {
|
||||
Repo::image(prompt, model, width, height)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
@@ -42,6 +42,13 @@ pub async fn download_youtube(url: &str, quality: &str) -> Result<DownloadResult
|
||||
DownloaderRepository::download_youtube(url, quality).await
|
||||
}
|
||||
|
||||
pub async fn download_youtube_merge(
|
||||
url: &str,
|
||||
quality: &str,
|
||||
) -> Result<DownloadResult, ScrapingError> {
|
||||
DownloaderRepository::fetch_youtube_merge(url, quality).await
|
||||
}
|
||||
|
||||
pub async fn download_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
DownloaderRepository::download_youtube_mp3(url).await
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
pub mod ai;
|
||||
pub mod anime;
|
||||
pub mod anime2;
|
||||
pub mod downloader;
|
||||
@@ -9,5 +8,3 @@ pub mod proxy;
|
||||
pub mod search;
|
||||
pub mod stalk;
|
||||
pub mod tools;
|
||||
pub mod uploader;
|
||||
pub mod weebs;
|
||||
@@ -18,44 +18,9 @@ pub async fn youtube(username: &str) -> Result<Value, ScrapingError> {
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
/// Mobile Legends player stalk.
|
||||
pub async fn mobile_legends(user_id: &str, zone_id: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_ml_stalk(user_id, zone_id)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
/// Free Fire player stalk.
|
||||
pub async fn free_fire(user_id: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_ff_stalk(user_id)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
/// Genshin Impact player stalk.
|
||||
pub async fn genshin(user_id: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_genshin_stalk(user_id)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
/// Twitter/X user stalk.
|
||||
pub async fn twitter(username: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_twitter_stalk(username)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
/// TikTok user stalk.
|
||||
pub async fn tiktok(username: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_tiktok_stalk(username)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
/// Instagram user stalk.
|
||||
pub async fn instagram(username: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_instagram_stalk(username)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
//! Application use-cases for the uploader.
|
||||
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::infrastructure::repository::uploader as Repo;
|
||||
use serde_json::Value;
|
||||
|
||||
pub fn upload(buffer: &[u8], file_name: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::save_file(buffer, file_name).map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
pub fn serve(file_name: &str) -> Result<(Vec<u8>, String), ScrapingError> {
|
||||
Repo::read_file(file_name).map_err(ScrapingError::Http)
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
//! Application use-cases for weebs (anime/manga info, waifu, whatanime).
|
||||
|
||||
use crate::domain::error::ScrapingError;
|
||||
use crate::infrastructure::repository::weebs as Repo;
|
||||
use serde_json::Value;
|
||||
|
||||
pub async fn anime_info(query: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_anime_info(query)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
pub async fn manga_info(query: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_manga_info(query)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
pub async fn whatanime(url: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_whatanime(url)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
pub async fn sfw_waifu(tag: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_sfw_waifu(tag)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
|
||||
pub async fn nsfw_waifu(tag: &str) -> Result<Value, ScrapingError> {
|
||||
Repo::fetch_nsfw_waifu(tag)
|
||||
.await
|
||||
.map_err(ScrapingError::Http)
|
||||
}
|
||||
@@ -1,272 +0,0 @@
|
||||
//! Infrastructure — AI endpoints via Pollinations.ai (free, no API key).
|
||||
//!
|
||||
//! Ported from Shirokami-API `scraper/ai/pollinationsai-{text,image,gemini}.js`.
|
||||
//! All three use the public Pollinations API:
|
||||
//! - text: POST https://text.pollinations.ai/openai (OpenAI-compatible)
|
||||
//! - gemini: POST https://text.pollinations.ai/openai with model=gemini
|
||||
//! - image: GET https://image.pollinations.ai/prompt/{prompt}
|
||||
//! Retry logic follows the source: try Bearer token (hardcoded in source),
|
||||
//! fall back to Referer when 401/403 (no key needed).
|
||||
|
||||
use crate::infrastructure::utils::http_client::http_client;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
const TEXT_URL: &str = "https://text.pollinations.ai/openai";
|
||||
const IMAGE_URL: &str = "https://image.pollinations.ai/prompt";
|
||||
// The source ships a hardcoded token; on 401/403 it falls back to Referer-only.
|
||||
// We skip the token and use Referer directly (anonymous tier) — same result,
|
||||
// no secret embedded in the binary.
|
||||
const REFERER: &str = "https://api.ryzumi.vip";
|
||||
|
||||
fn ua() -> &'static str {
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36"
|
||||
}
|
||||
|
||||
fn build_messages(system: &str, text: &str, image_data_url: Option<String>) -> Vec<Value> {
|
||||
let mut messages = Vec::new();
|
||||
if !system.is_empty() {
|
||||
messages.push(json!({"role": "system", "content": system}));
|
||||
}
|
||||
if let Some(img) = image_data_url {
|
||||
let mut content = Vec::new();
|
||||
if !text.is_empty() {
|
||||
content.push(json!({"type": "text", "text": text}));
|
||||
}
|
||||
content.push(json!({"type": "image_url", "image_url": {"url": img}}));
|
||||
messages.push(json!({"role": "user", "content": content}));
|
||||
} else {
|
||||
messages.push(json!({"role": "user", "content": text}));
|
||||
}
|
||||
messages
|
||||
}
|
||||
|
||||
/// Chat via Pollinations (OpenAI-compatible).
|
||||
/// Default model "openai" (the source's "gpt-5-nano" no longer exists on
|
||||
/// Pollinations and 404s; fallback to "openai" on any error, incl. 404).
|
||||
pub async fn chat(text: String, prompt: String, model: Option<String>) -> Result<Value, String> {
|
||||
let model = model.unwrap_or_else(|| "openai".to_string());
|
||||
let system = if prompt.trim().is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}\nConversation history:", prompt)
|
||||
};
|
||||
let messages = build_messages(&system, &text, None);
|
||||
|
||||
let body = json!({
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"referrer": REFERER,
|
||||
});
|
||||
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.post(TEXT_URL)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Referer", REFERER)
|
||||
.header("User-Agent", ua())
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
// Match source: on failure retry with model "openai"
|
||||
let status = resp.status();
|
||||
if status == reqwest::StatusCode::UNAUTHORIZED
|
||||
|| status == reqwest::StatusCode::FORBIDDEN
|
||||
|| status == reqwest::StatusCode::NOT_FOUND
|
||||
|| status == reqwest::StatusCode::PAYMENT_REQUIRED
|
||||
{
|
||||
let body2 = json!({
|
||||
"model": "openai",
|
||||
"messages": messages,
|
||||
"referrer": REFERER,
|
||||
});
|
||||
let resp2 = http_client()
|
||||
.client()
|
||||
.post(TEXT_URL)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Referer", REFERER)
|
||||
.header("User-Agent", ua())
|
||||
.json(&body2)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP retry: {e}"))?;
|
||||
return parse_text_response(resp2).await;
|
||||
}
|
||||
return Err(format!("API error {status}"));
|
||||
}
|
||||
|
||||
parse_text_response(resp).await
|
||||
}
|
||||
|
||||
/// Gemini chat via Pollinations (model "gemini").
|
||||
pub async fn gemini(
|
||||
text: String,
|
||||
prompt: String,
|
||||
image_url: Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
let system = if prompt.trim().is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{}\nConversation history:", prompt)
|
||||
};
|
||||
|
||||
// If an image URL is given, fetch it and convert to a data URL (source behavior).
|
||||
let image_data_url = match &image_url {
|
||||
Some(url) if !url.trim().is_empty() => Some(fetch_data_url(url).await?),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let messages = build_messages(&system, &text, image_data_url);
|
||||
|
||||
let body = json!({
|
||||
"model": "gemini",
|
||||
"messages": messages,
|
||||
"referrer": REFERER,
|
||||
});
|
||||
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.post(TEXT_URL)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Referer", REFERER)
|
||||
.header("User-Agent", ua())
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
if status == reqwest::StatusCode::UNAUTHORIZED
|
||||
|| status == reqwest::StatusCode::FORBIDDEN
|
||||
|| status == reqwest::StatusCode::NOT_FOUND
|
||||
|| status == reqwest::StatusCode::PAYMENT_REQUIRED
|
||||
{
|
||||
let body2 = json!({
|
||||
"model": "openai",
|
||||
"messages": messages,
|
||||
"referrer": REFERER,
|
||||
});
|
||||
let resp2 = http_client()
|
||||
.client()
|
||||
.post(TEXT_URL)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Referer", REFERER)
|
||||
.header("User-Agent", ua())
|
||||
.json(&body2)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP retry: {e}"))?;
|
||||
return parse_text_response(resp2).await;
|
||||
}
|
||||
return Err(format!("API error {status}"));
|
||||
}
|
||||
|
||||
parse_text_response(resp).await
|
||||
}
|
||||
|
||||
/// Image generation via Pollinations. Returns raw image bytes.
|
||||
pub async fn image(
|
||||
prompt: String,
|
||||
model: String,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let model = if model.is_empty() {
|
||||
"flux".to_string()
|
||||
} else {
|
||||
model
|
||||
};
|
||||
let valid = ["flux", "kontext", "turbo"];
|
||||
if !valid.contains(&model.as_str()) {
|
||||
return Err(format!(
|
||||
"Invalid model. Available models: {}",
|
||||
valid.join(", ")
|
||||
));
|
||||
}
|
||||
|
||||
let url = format!(
|
||||
"{}/{}?model={}&width={}&height={}&nologo=true&private=false&enhance=false&safe=false&referrer={}",
|
||||
IMAGE_URL,
|
||||
urlencode(&prompt),
|
||||
model,
|
||||
width,
|
||||
height,
|
||||
urlencode(REFERER)
|
||||
);
|
||||
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.get(&url)
|
||||
.header("Referer", REFERER)
|
||||
.header("User-Agent", ua())
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("Image API error {}", resp.status()));
|
||||
}
|
||||
|
||||
let bytes = resp.bytes().await.map_err(|e| format!("Body: {e}"))?;
|
||||
Ok(bytes.to_vec())
|
||||
}
|
||||
|
||||
async fn parse_text_response(resp: reqwest::Response) -> Result<Value, String> {
|
||||
let data: Value = resp.json().await.map_err(|e| format!("JSON: {e}"))?;
|
||||
let content = data
|
||||
.pointer("/choices/0/message/content")
|
||||
.or_else(|| data.get("message"))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| Value::String("No response from assistant.".into()));
|
||||
Ok(json!({"success": true, "result": content}))
|
||||
}
|
||||
|
||||
async fn fetch_data_url(url: &str) -> Result<String, String> {
|
||||
if url.starts_with("data:") {
|
||||
return Ok(url.to_string());
|
||||
}
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.get(url)
|
||||
.header("User-Agent", ua())
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("Fetch image failed: {e}"))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("Fetch image failed: {}", resp.status()));
|
||||
}
|
||||
let ct = resp
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("image/jpeg")
|
||||
.to_string();
|
||||
let ext = ct
|
||||
.split('/')
|
||||
.nth(1)
|
||||
.map(|s| s.split('+').next().unwrap_or("jpeg").to_string())
|
||||
.unwrap_or_else(|| "jpeg".to_string());
|
||||
let bytes = resp.bytes().await.map_err(|e| format!("Body: {e}"))?;
|
||||
use base64::Engine;
|
||||
Ok(format!(
|
||||
"data:image/{};base64,{}",
|
||||
ext,
|
||||
base64::engine::general_purpose::STANDARD.encode(bytes)
|
||||
))
|
||||
}
|
||||
|
||||
fn urlencode(s: &str) -> String {
|
||||
let mut out = String::new();
|
||||
for b in s.bytes() {
|
||||
match b {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
||||
out.push(b as char)
|
||||
}
|
||||
_ => out.push_str(&format!("%{:02X}", b)),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::Duration;
|
||||
|
||||
use aes::cipher::{BlockDecrypt, KeyInit};
|
||||
@@ -511,6 +513,14 @@ impl DownloaderRepository {
|
||||
fetch_youtube_mp4(url, quality).await
|
||||
}
|
||||
|
||||
/// YouTube video + audio merged server-side into a single MP4.
|
||||
pub async fn fetch_youtube_merge(
|
||||
url: &str,
|
||||
quality: &str,
|
||||
) -> Result<DownloadResult, ScrapingError> {
|
||||
fetch_youtube_merge_free(url, quality).await
|
||||
}
|
||||
|
||||
/// YouTube to MP3 via ydlp.yard.id.
|
||||
pub async fn download_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
fetch_youtube_mp3(url).await
|
||||
@@ -844,12 +854,26 @@ async fn detect_media_type(url: &str) -> MediaType {
|
||||
/// SnapSave parser — extracts Instagram/Facebook media via snapsave.app.
|
||||
/// Uses downr.org as fallback for robustness, then Playwright browser scraping.
|
||||
pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
// Validate Instagram/Facebook URL
|
||||
let valid_fb = regex::Regex::new(r"https?://(web\.|www\.|m\.)(facebook|fb)\.(com|watch)\S+")
|
||||
// Validate Instagram/Facebook URL.
|
||||
//
|
||||
// Facebook link shapes (all must pass):
|
||||
// https://www.facebook.com/watch/?v=123
|
||||
// https://fb.watch/abc123/
|
||||
// https://m.facebook.com/video.php?v=123
|
||||
// https://www.facebook.com/100012345678901/videos/1234567890
|
||||
// https://www.facebook.com/reel/123
|
||||
// https://www.facebook.com/story.php?story_fbid=123
|
||||
// https://web.facebook.com/...
|
||||
// NOTE: the old regex required a `www.|m.|web.` subdomain AND ended the
|
||||
// host capture with `(facebook|fb)\.(com|watch)` — that rejected
|
||||
// `facebook.com/watch...` (no subdomain) and `fb.watch/...` entirely.
|
||||
let fb_host = r"(?:[a-z0-9-]*\.)?(?:facebook|fb)\.(?:com|watch)";
|
||||
let valid_fb = regex::Regex::new(&format!(r"https?://{}(?:/|$)", fb_host))
|
||||
.unwrap()
|
||||
.is_match(url);
|
||||
let valid_ig =
|
||||
regex::Regex::new(r"https?://(www\.)?instagram\.com/(p|reel|reels|tv|stories)/\S+")
|
||||
let valid_ig = url.contains("instagram.com")
|
||||
|| url.contains("threads.net")
|
||||
|| regex::Regex::new(r"https?://(www\.)?instagram\.com/[^\s]+")
|
||||
.unwrap()
|
||||
.is_match(url);
|
||||
|
||||
@@ -859,7 +883,24 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
||||
));
|
||||
}
|
||||
|
||||
// Try downr.org first
|
||||
// Try well-known scrapers in order — Playwright is a LAST resort (heavy,
|
||||
// slow, and its naive response-harvesting returns many broken/404 links
|
||||
// mixed with the one good URL, which is exactly the "link exists but can't
|
||||
// download" bug we've seen).
|
||||
//
|
||||
// 1) yt-dlp facebook extractor — authoritative, returns direct fbcdn
|
||||
// progressive URLs with REAL title/format metadata and only the working
|
||||
// scales (sd + hd, no junk). Also handles /reel, /watch, /videos,
|
||||
// story.php, video.php, fb.watch URL shapes.
|
||||
if let Ok(yt) = run_ytdlp_json(url, &["-f", "bestvideo+bestaudio/best"]).await {
|
||||
if let Some(res) = ytdlp_to_download_result(&yt) {
|
||||
if !res.media.is_empty() {
|
||||
return Ok(res);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2) downr.org all-in-one (same provider as Shirokami's /fbdl)
|
||||
match fetch_all_in_one(url).await {
|
||||
Ok(result) if !result.media.is_empty() => return Ok(result),
|
||||
Ok(_) => {}
|
||||
@@ -871,7 +912,7 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: use Playwright browser scraping to extract video URLs
|
||||
// 3) Playwright browser scraping to extract video URLs
|
||||
let platform = if valid_ig { "instagram" } else { "facebook" };
|
||||
let data = run_playwright_scraper(url, platform).await?;
|
||||
|
||||
@@ -889,17 +930,25 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
||||
let media_arr = data.get("media").and_then(|v| v.as_array());
|
||||
if let Some(medias) = media_arr {
|
||||
for m in medias {
|
||||
let item = MediaItem {
|
||||
url: m
|
||||
let u = m
|
||||
.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string(),
|
||||
.to_string();
|
||||
// Drop Playwright-harvested links that are actually broken
|
||||
// byte-range chunks (fbcdn URLs with bytestart=…byteend=… that
|
||||
// return tiny 200 responses — they 404 or return an HTML error
|
||||
// page when fetched as a whole), and the empty-string url.
|
||||
if u.is_empty() || u.contains("bytestart=") {
|
||||
continue;
|
||||
}
|
||||
let item = MediaItem {
|
||||
url: u,
|
||||
quality: None,
|
||||
file_type: m.get("ext").and_then(|v| v.as_str()).and_then(|e| match e {
|
||||
"mp4" | "m3u8" => Some(MediaType::Video),
|
||||
"mp3" | "m4a" => Some(MediaType::Audio),
|
||||
_ => Some(MediaType::Video),
|
||||
file_type: m.get("ext").and_then(|v| v.as_str()).map(|e| match e {
|
||||
"mp4" | "m3u8" => MediaType::Video,
|
||||
"mp3" | "m4a" => MediaType::Audio,
|
||||
_ => MediaType::Video,
|
||||
}),
|
||||
extension: m.get("ext").and_then(|v| v.as_str()).map(|s| s.to_string()),
|
||||
thumbnail: None,
|
||||
@@ -913,6 +962,13 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
||||
}
|
||||
}
|
||||
|
||||
// De-duplicate by URL (Playwright harvests the same CDN file at many
|
||||
// byte-ranges, and the primary URL appears multiple times).
|
||||
{
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
result.media.retain(|m| seen.insert(m.url.clone()));
|
||||
}
|
||||
|
||||
if result.media.is_empty() {
|
||||
return Ok(DownloadResult::error(format!(
|
||||
"Failed to extract media from {} — server IP may be blocked by anti-bot protection",
|
||||
@@ -922,6 +978,122 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Convert a yt-dlp --dump-single-json result into a DownloadResult with the
|
||||
/// direct progressive/adaptive media URLs (only entries that carry a URL).
|
||||
/// Used by the Facebook downloader as the primary provider.
|
||||
fn ytdlp_to_download_result(data: &serde_json::Value) -> Option<DownloadResult> {
|
||||
let title = data
|
||||
.get("title")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
let mut result = DownloadResult::success(title);
|
||||
result.provider = Some("yt-dlp".to_string());
|
||||
result.thumbnail = data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
result.duration = data
|
||||
.get("duration")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|d| format!("{}s", d));
|
||||
|
||||
let formats = data.get("formats").and_then(|v| v.as_array());
|
||||
let requested_formats = data.get("requested_formats").and_then(|v| v.as_array());
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
|
||||
// `-f bestvideo+bestaudio/best` puts the merged URLs in requested_formats.
|
||||
// `best` single format lands in top-level url + formats[0].
|
||||
for list in [requested_formats, formats].into_iter().flatten() {
|
||||
for f in list {
|
||||
let Some(furl) = f.get("url").and_then(|v| v.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
if !furl.starts_with("http") || !seen.insert(furl.to_string()) {
|
||||
continue;
|
||||
}
|
||||
let proto = f.get("protocol").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if proto.contains("m3u8")
|
||||
|| f.get("vcodec").and_then(|v| v.as_str()) == Some("none")
|
||||
&& f.get("acodec").and_then(|v| v.as_str()) != Some("none")
|
||||
{
|
||||
// Skip HLS + audio-only; Facebook's progressive mp4s are direct.
|
||||
continue;
|
||||
}
|
||||
let ext: String = f
|
||||
.get("ext")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("mp4")
|
||||
.to_string();
|
||||
result.media.push(MediaItem {
|
||||
url: furl.to_string(),
|
||||
quality: f
|
||||
.get("height")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|h| format!("{}p", h))
|
||||
.or_else(|| {
|
||||
f.get("format_note")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
}),
|
||||
file_type: Some(if ext == "mp4" || ext == "webm" || ext == "mkv" {
|
||||
MediaType::Video
|
||||
} else {
|
||||
MediaType::File
|
||||
}),
|
||||
extension: Some(ext),
|
||||
thumbnail: result.thumbnail.clone(),
|
||||
file_size: f
|
||||
.get("filesize")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(format_filesize),
|
||||
size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
|
||||
frame_width: f
|
||||
.get("width")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|w| w.to_string()),
|
||||
frame_height: f
|
||||
.get("height")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|h| h.to_string()),
|
||||
note: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to the merged top-level URL (bestvideo+bestaudio sets it).
|
||||
if result.media.is_empty() {
|
||||
if let Some(dl_url) = data.get("url").and_then(|v| v.as_str()) {
|
||||
if dl_url.starts_with("http") {
|
||||
result.media.push(MediaItem {
|
||||
url: dl_url.to_string(),
|
||||
quality: None,
|
||||
file_type: Some(MediaType::Video),
|
||||
extension: data
|
||||
.get("ext")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| Some("mp4".to_string())),
|
||||
thumbnail: None,
|
||||
file_size: data
|
||||
.get("filesize")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(format_filesize),
|
||||
size_bytes: data.get("filesize").and_then(|v| v.as_u64()),
|
||||
frame_width: None,
|
||||
frame_height: None,
|
||||
note: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if result.media.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(result)
|
||||
}
|
||||
}
|
||||
/// TikTok via embed-page scraping (primary method).
|
||||
/// Scrapes `https://www.tiktok.com/embed/v2/{video_id}` HTML and extracts the
|
||||
/// direct `v16m.tiktokcdn.com` MP4 URL from the `<video data-testid="play-video">` tag.
|
||||
@@ -1402,7 +1574,11 @@ pub async fn fetch_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingErro
|
||||
data.get("abr").and_then(|v| v.as_u64()).unwrap_or(128)
|
||||
)),
|
||||
file_type: Some(MediaType::Audio),
|
||||
extension: Some("mp3".to_string()),
|
||||
extension: data
|
||||
.get("ext")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| Some("webm".to_string())),
|
||||
thumbnail: data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
@@ -1425,11 +1601,116 @@ pub async fn fetch_youtube_mp4(url: &str, quality: &str) -> Result<DownloadResul
|
||||
extract_youtube_id(url)
|
||||
.ok_or_else(|| ScrapingError::Http("Invalid YouTube URL".to_string()))?;
|
||||
|
||||
let fmt = format!(
|
||||
"bestvideo[height<={}]/best+bestaudio/best",
|
||||
quality.trim_end_matches('p')
|
||||
let q = quality.trim_end_matches('p');
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// 1) PREFER a PROGRESSIVE format (video + audio in ONE file).
|
||||
// The `android` player client exposes progressive itags (18/22) that
|
||||
// are NOT rate-limited from datacenter IPs (verified 2026-09-05:
|
||||
// itag 18 downloads at full speed, while audio-only streams crawl).
|
||||
// Fall back to adaptive (bestvideo+bestaudio) below when no
|
||||
// progressive format exists at the requested height.
|
||||
// ---------------------------------------------------------------
|
||||
let prog_fmt = format!("best[height<={}][ext=mp4]/best[height<={}]", q, q);
|
||||
if let Ok(prog_data) = run_ytdlp_json(
|
||||
url,
|
||||
&[
|
||||
"--extractor-args",
|
||||
"youtube:player_client=android",
|
||||
"-f",
|
||||
&prog_fmt,
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
let prog_url = prog_data
|
||||
.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
let prog_vcodec = prog_data
|
||||
.get("vcodec")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("none");
|
||||
let prog_acodec = prog_data
|
||||
.get("acodec")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("none");
|
||||
let has_audio = prog_acodec != "none" && !prog_acodec.is_empty();
|
||||
let has_video = prog_vcodec != "none" && !prog_vcodec.is_empty();
|
||||
if let Some(purl) = prog_url {
|
||||
if has_audio && has_video && purl.starts_with("http") {
|
||||
let height = prog_data
|
||||
.get("height")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(360);
|
||||
// Only use the progressive format if it actually meets the
|
||||
// requested quality. `best[height<=Q][ext=mp4]` can return a
|
||||
// lower-height progressive (e.g. 360p when 720p was requested)
|
||||
// because no 720p progressive exists; in that case fall through
|
||||
// to the adaptive (bestvideo+bestaudio) path so the caller
|
||||
// still gets the requested resolution.
|
||||
let q_num: u64 = q.parse().unwrap_or(0);
|
||||
if q_num != 0 && height < q_num {
|
||||
// fall through to adaptive below
|
||||
} else {
|
||||
let ext = prog_data
|
||||
.get("ext")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("mp4");
|
||||
let mut result = DownloadResult::success(
|
||||
prog_data
|
||||
.get("title")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string()),
|
||||
);
|
||||
let data = run_ytdlp_json(url, &["-f", &fmt]).await?;
|
||||
result.author = prog_data
|
||||
.get("uploader")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
result.thumbnail = prog_data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
result.duration = prog_data
|
||||
.get("duration")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|d| format!("{}s", d));
|
||||
result.provider = Some("yt-dlp".to_string());
|
||||
result.media.push(MediaItem {
|
||||
url: purl,
|
||||
quality: Some(format!("{}p", height)),
|
||||
file_type: Some(MediaType::Video),
|
||||
extension: Some(ext.to_string()),
|
||||
thumbnail: prog_data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string()),
|
||||
file_size: prog_data
|
||||
.get("filesize")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(format_filesize),
|
||||
size_bytes: prog_data.get("filesize").and_then(|v| v.as_u64()),
|
||||
frame_width: prog_data
|
||||
.get("width")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|w| w.to_string()),
|
||||
frame_height: prog_data
|
||||
.get("height")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|h| h.to_string()),
|
||||
note: Some("Progressive MP4 — video and audio in one file".to_string()),
|
||||
});
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let fmt = format!(
|
||||
"bestvideo[height<={}]+bestaudio/best[height<={}]/best",
|
||||
q, q
|
||||
);
|
||||
let data = run_ytdlp_json(url, &["-f", &fmt, "--merge-output-format", "mp4"]).await?;
|
||||
|
||||
let title = data
|
||||
.get("title")
|
||||
@@ -1455,23 +1736,101 @@ pub async fn fetch_youtube_mp4(url: &str, quality: &str) -> Result<DownloadResul
|
||||
result.provider = Some("yt-dlp".to_string());
|
||||
|
||||
let formats = data.get("formats").and_then(|v| v.as_array());
|
||||
let requested_formats = data.get("requested_formats").and_then(|v| v.as_array());
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
|
||||
if let Some(fmts) = formats {
|
||||
// When yt-dlp uses a merged format (bestvideo+bestaudio), the actual
|
||||
// download URLs live in `requested_formats` (video + audio separate URLs).
|
||||
// The top-level `url` is empty and `formats[]` is the FULL list of all
|
||||
// individual format IDs, not just the selected ones.
|
||||
if let Some(rf) = requested_formats {
|
||||
for f in rf {
|
||||
if let (Some(furl), Some(ext_val)) = (
|
||||
f.get("url").and_then(|v| v.as_str()),
|
||||
f.get("ext").and_then(|v| v.as_str()),
|
||||
) {
|
||||
let proto = f.get("protocol").and_then(|v| v.as_str()).unwrap_or("");
|
||||
// Skip mhtml/storyboard and m3u8/hls streams
|
||||
if ext_val == "mhtml" || proto.contains("m3u8") {
|
||||
continue;
|
||||
}
|
||||
let url_string = furl.to_string();
|
||||
if !url_string.starts_with("http") {
|
||||
continue;
|
||||
}
|
||||
if seen.insert(url_string.clone()) {
|
||||
let is_video = f
|
||||
.get("vcodec")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|c| c != "none")
|
||||
.unwrap_or(false);
|
||||
let is_audio_only = f
|
||||
.get("acodec")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|c| c != "none")
|
||||
.unwrap_or(false);
|
||||
let fmt_type = if is_video && is_audio_only {
|
||||
MediaType::Video
|
||||
} else if is_audio_only {
|
||||
MediaType::Audio
|
||||
} else if is_video {
|
||||
MediaType::Video
|
||||
} else {
|
||||
MediaType::File
|
||||
};
|
||||
let q = format!("{}p", f.get("height").and_then(|v| v.as_u64()).unwrap_or(0));
|
||||
result.media.push(MediaItem {
|
||||
url: url_string,
|
||||
quality: Some(q),
|
||||
file_type: Some(fmt_type),
|
||||
extension: Some(ext_val.to_string()),
|
||||
thumbnail: data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string()),
|
||||
file_size: f
|
||||
.get("filesize")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(format_filesize),
|
||||
size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
|
||||
frame_width: f
|
||||
.get("width")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|w| w.to_string()),
|
||||
frame_height: f
|
||||
.get("height")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|h| h.to_string()),
|
||||
note: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if let Some(fmts) = formats {
|
||||
for f in fmts {
|
||||
if let (Some(furl), Some(ext_val)) = (
|
||||
f.get("url").and_then(|v| v.as_str()),
|
||||
f.get("ext").and_then(|v| v.as_str()),
|
||||
) {
|
||||
if ext_val == "mhtml" {
|
||||
// Skip storyboard, mhtml, m3u8/hls, and audio-only formats.
|
||||
// We only want formats that have both video and audio.
|
||||
let proto = f.get("protocol").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if ext_val == "mhtml" || proto.contains("m3u8") {
|
||||
continue;
|
||||
}
|
||||
// Skip audio-only formats (vcodec == "none", acodec != "none")
|
||||
let vcodec = f.get("vcodec").and_then(|v| v.as_str()).unwrap_or("none");
|
||||
let acodec = f.get("acodec").and_then(|v| v.as_str()).unwrap_or("none");
|
||||
if vcodec == "none" && acodec != "none" {
|
||||
continue; // audio-only
|
||||
}
|
||||
let url_string = furl.to_string();
|
||||
if !url_string.starts_with("http") {
|
||||
continue;
|
||||
}
|
||||
if seen.insert(url_string.clone()) {
|
||||
let fmt_type = if ext_val == "mp4" || ext_val == "webm" || ext_val == "mkv" {
|
||||
MediaType::Video
|
||||
} else if ext_val == "mp3" || ext_val == "m4a" || ext_val == "webm" {
|
||||
MediaType::Audio
|
||||
} else {
|
||||
MediaType::File
|
||||
};
|
||||
@@ -1536,6 +1895,252 @@ pub async fn fetch_youtube_mp4(url: &str, quality: &str) -> Result<DownloadResul
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// YouTube video + audio SERVER-SIDE MERGE (produces a single MP4 with audio).
|
||||
//
|
||||
// Modern YouTube videos (AV1/VP9, 720p+) only expose separate video-only and
|
||||
// audio-only streams. `fetch_youtube_mp4` returns those two URLs — the caller
|
||||
// would have to merge them manually. This endpoint downloads both streams and
|
||||
// remuxes them into ONE MP4 with `ffmpeg`, stored under the uploads dir and
|
||||
// served via GET /file/yt_merge/{name}. Files older than 2h are cleaned up on
|
||||
// each new merge request to bound disk usage.
|
||||
// ============================================================================
|
||||
pub async fn fetch_youtube_merge_free(
|
||||
url: &str,
|
||||
quality: &str,
|
||||
) -> Result<DownloadResult, ScrapingError> {
|
||||
let video_id = extract_youtube_id(url)
|
||||
.ok_or_else(|| ScrapingError::Http("Invalid YouTube URL".to_string()))?;
|
||||
|
||||
let q = quality.trim_end_matches('p');
|
||||
let fmt = format!(
|
||||
"bestvideo[height<={}]+bestaudio/best[height<={}]/best",
|
||||
q, q
|
||||
);
|
||||
let data = run_ytdlp_json(url, &["-f", &fmt, "--merge-output-format", "mp4"]).await?;
|
||||
|
||||
let title = data
|
||||
.get("title")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
let author = data
|
||||
.get("uploader")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
let thumbnail = data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
let duration = data
|
||||
.get("duration")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|d| format!("{}s", d));
|
||||
|
||||
// Locate the video-only and audio-only URLs from requested_formats.
|
||||
let requested_formats = data
|
||||
.get("requested_formats")
|
||||
.and_then(|v| v.as_array())
|
||||
.ok_or_else(|| ScrapingError::Http("No requested_formats from yt-dlp".to_string()))?;
|
||||
|
||||
let mut video_url: Option<String> = None;
|
||||
let mut audio_url: Option<String> = None;
|
||||
let mut video_ext = "mp4";
|
||||
let mut audio_ext = "webm";
|
||||
for f in requested_formats {
|
||||
let vcodec = f.get("vcodec").and_then(|v| v.as_str()).unwrap_or("none");
|
||||
let acodec = f.get("acodec").and_then(|v| v.as_str()).unwrap_or("none");
|
||||
let f_url = f.get("url").and_then(|v| v.as_str());
|
||||
let proto = f.get("protocol").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if proto.contains("m3u8") {
|
||||
continue; // HLS streams are not directly downloadable via reqwest
|
||||
}
|
||||
if vcodec != "none" && acodec == "none" {
|
||||
if let Some(u) = f_url {
|
||||
video_url = Some(u.to_string());
|
||||
video_ext = f.get("ext").and_then(|v| v.as_str()).unwrap_or("mp4");
|
||||
}
|
||||
} else if vcodec == "none" && acodec != "none" {
|
||||
if let Some(u) = f_url {
|
||||
audio_url = Some(u.to_string());
|
||||
audio_ext = f.get("ext").and_then(|v| v.as_str()).unwrap_or("webm");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let video_url = video_url
|
||||
.ok_or_else(|| ScrapingError::Http("No video stream URL from yt-dlp".to_string()))?;
|
||||
let audio_url = audio_url
|
||||
.ok_or_else(|| ScrapingError::Http("No audio stream URL from yt-dlp".to_string()))?;
|
||||
|
||||
// --- download both streams to the merge temp dir ---
|
||||
let merge_dir = PathBuf::from("/var/lib/scraper/uploads/yt_merge");
|
||||
tokio::fs::create_dir_all(&merge_dir)
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("create merge dir failed: {e}")))?;
|
||||
|
||||
// Cleanup stale merged files (> 2h) before writing anything new.
|
||||
cleanup_stale_merges(&merge_dir).await;
|
||||
|
||||
let job_id = uuid::Uuid::new_v4().simple().to_string();
|
||||
let vpath = merge_dir.join(format!("{video_id}_{job_id}_v.{video_ext}"));
|
||||
let apath = merge_dir.join(format!("{video_id}_{job_id}_a.{audio_ext}"));
|
||||
let outpath = merge_dir.join(format!("{video_id}_{job_id}_merged.mp4"));
|
||||
|
||||
download_to_file(&video_url, &vpath).await?;
|
||||
download_to_file(&audio_url, &apath).await?;
|
||||
|
||||
// --- merge with ffmpeg ---
|
||||
let merge_result = tokio::task::spawn_blocking({
|
||||
let vpath = vpath.clone();
|
||||
let apath = apath.clone();
|
||||
let outpath = outpath.clone();
|
||||
move || merge_with_ffmpeg(&vpath, &apath, &outpath)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("ffmpeg join failed: {e}")))?;
|
||||
|
||||
match merge_result {
|
||||
Ok(()) => {}
|
||||
Err(e) => {
|
||||
let _ = tokio::fs::remove_file(&vpath).await;
|
||||
let _ = tokio::fs::remove_file(&apath).await;
|
||||
return Err(ScrapingError::Http(format!("ffmpeg merge failed: {e}")));
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up the transient inputs now that the merged file exists.
|
||||
let _ = tokio::fs::remove_file(&vpath).await;
|
||||
let _ = tokio::fs::remove_file(&apath).await;
|
||||
|
||||
let file_size = tokio::fs::metadata(&outpath).await.ok().map(|m| m.len());
|
||||
|
||||
let file_name = outpath
|
||||
.file_name()
|
||||
.map(|s| s.to_string_lossy().to_string())
|
||||
.unwrap_or_default();
|
||||
let public_url = format!("/file/yt_merge/{file_name}");
|
||||
|
||||
let mut result = DownloadResult::success(title);
|
||||
result.author = author;
|
||||
result.thumbnail = thumbnail.clone();
|
||||
result.duration = duration;
|
||||
result.provider = Some("yt-dlp+ffmpeg".to_string());
|
||||
result.media.push(MediaItem {
|
||||
url: public_url,
|
||||
quality: Some(format!("{}p merged", q)),
|
||||
file_type: Some(MediaType::Video),
|
||||
extension: Some("mp4".to_string()),
|
||||
thumbnail: thumbnail.clone(),
|
||||
file_size: file_size.map(format_filesize),
|
||||
size_bytes: file_size,
|
||||
frame_width: None,
|
||||
frame_height: None,
|
||||
note: Some(
|
||||
"Merged server-side: single MP4 file containing both video and audio".to_string(),
|
||||
),
|
||||
});
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Download `url` to `path` with a generous timeout. The yt CDN stream URLs are
|
||||
/// signed and short-lived, so we must download promptly and cannot retry after
|
||||
/// the URL expires.
|
||||
async fn download_to_file(url: &str, path: &Path) -> Result<(), ScrapingError> {
|
||||
// Dedicated client with a longer timeout than the shared 30s one: merged
|
||||
// files are often tens of MB and the yt CDN can be slow from a datacenter.
|
||||
// .no_gzip()/.no_brotli()/.no_deflate() disable reqwest's auto-decompress
|
||||
// so we can write raw bytes — some CDN responses are mis-labelled and fail
|
||||
// the auto-decode step.
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(300))
|
||||
.connect_timeout(Duration::from_secs(20))
|
||||
.no_gzip()
|
||||
.no_brotli()
|
||||
.no_deflate()
|
||||
.build()
|
||||
.map_err(|e| ScrapingError::Http(format!("stream client build failed: {e}")))?;
|
||||
let resp = client
|
||||
.get(url)
|
||||
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
.header("Accept-Encoding", "identity")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("stream download failed: {e}")))?;
|
||||
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
return Err(ScrapingError::Http(format!(
|
||||
"stream download HTTP {status}"
|
||||
)));
|
||||
}
|
||||
|
||||
let bytes = resp
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("stream body read failed: {e}")))?;
|
||||
|
||||
tokio::fs::write(path, bytes)
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("stream write failed: {e}")))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run ffmpeg to remux the video and audio streams into a single MP4.
|
||||
/// `-c:v copy` avoids a full re-encode (fast); audio is transcoded to AAC so
|
||||
/// the output plays everywhere. `+faststart` moves the moov atom to the front
|
||||
/// so the file streams immediately.
|
||||
fn merge_with_ffmpeg(video: &Path, audio: &Path, out: &Path) -> Result<(), String> {
|
||||
let status = Command::new("/bin/ffmpeg")
|
||||
.args([
|
||||
"-y",
|
||||
"-v",
|
||||
"error",
|
||||
"-i",
|
||||
video.to_str().unwrap_or_default(),
|
||||
"-i",
|
||||
audio.to_str().unwrap_or_default(),
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
"-shortest",
|
||||
out.to_str().unwrap_or_default(),
|
||||
])
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.status()
|
||||
.map_err(|e| format!("spawn ffmpeg failed: {e}"))?;
|
||||
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("ffmpeg exited with {status:?}"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove merged output + transient files older than 2 hours. Bounded, best
|
||||
/// effort — never fails the request.
|
||||
async fn cleanup_stale_merges(dir: &Path) {
|
||||
let deadline = std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 60 * 60);
|
||||
let Ok(mut entries) = tokio::fs::read_dir(dir).await else {
|
||||
return;
|
||||
};
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let Ok(meta) = entry.metadata().await else {
|
||||
continue;
|
||||
};
|
||||
let Ok(modified) = meta.modified() else {
|
||||
continue;
|
||||
};
|
||||
if modified < deadline {
|
||||
let _ = tokio::fs::remove_file(entry.path()).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Spotify downloaders
|
||||
// ============================================================================
|
||||
|
||||
@@ -2804,56 +3409,131 @@ pub async fn fetch_terabox(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
|
||||
let surl = surl.ok_or_else(|| ScrapingError::Http("SURL not found.".to_string()))?;
|
||||
|
||||
// Call local TeraBox resolver (Playwright + residential proxy + cookies)
|
||||
let resolver_url = std::env::var("TERABOX_RESOLVER_URL")
|
||||
.unwrap_or_else(|_| "http://127.0.0.1:4092".to_string());
|
||||
let resolve_url = format!("{}/resolve?surl={}", resolver_url, surl);
|
||||
|
||||
let client = http_client();
|
||||
let resp = client
|
||||
.client()
|
||||
.get("https://tera2.sylyt93.workers.dev/info")
|
||||
.query(&[("s", &surl)])
|
||||
.header("origin", "https://www.kauruka.com")
|
||||
.header("referer", "https://www.kauruka.com/")
|
||||
.get(&resolve_url)
|
||||
.header(USER_AGENT, "Mozilla/5.0")
|
||||
.timeout(Duration::from_secs(15))
|
||||
.timeout(Duration::from_secs(60))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("TeraBox info fetch failed: {}", e)))?
|
||||
.json::<serde_json::Value>()
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("TeraBox JSON parse failed: {}", e)))?;
|
||||
.map_err(|e| ScrapingError::Http(format!("TeraBox resolver fetch failed: {}", e)))?;
|
||||
|
||||
if let Some(download_url) = resp.get("url").and_then(|v| v.as_str()) {
|
||||
let mut result = DownloadResult::success(
|
||||
resp.get("name")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string()),
|
||||
);
|
||||
result.provider = Some("terabox".to_string());
|
||||
result.thumbnail = resp
|
||||
.get("img")
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Ok(DownloadResult::error(format!(
|
||||
"TeraBox resolver returned HTTP {}: {}",
|
||||
status, body
|
||||
)));
|
||||
}
|
||||
|
||||
let data: serde_json::Value = resp
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| ScrapingError::Http(format!("TeraBox resolver JSON parse failed: {}", e)))?;
|
||||
|
||||
if let Some(error) = data.get("error").and_then(|v| v.as_str()) {
|
||||
return Ok(DownloadResult::error(error));
|
||||
}
|
||||
|
||||
let file_name = data
|
||||
.get("file_name")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
|
||||
let mut result = DownloadResult::success(file_name.clone());
|
||||
result.provider = Some("terabox".to_string());
|
||||
result.thumbnail = data
|
||||
.get("thumbnail")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
|
||||
// Primary download link.
|
||||
//
|
||||
// IMPORTANT: the resolver's download_link is a raw terabox dlink that
|
||||
// 403s with {"error_code":31045,"user not exists"} when fetched WITHOUT
|
||||
// cookies — even from residential IPs. So we rewrite the public URL to
|
||||
// OUR OWN /proxy/terabox streaming endpoint (which forwards the resolver's
|
||||
// cookie-authenticated stream through the public API). This is the only
|
||||
// way users get a link that actually downloads.
|
||||
// The public API base that users can reach. `CONFIG.urls.site_url` is the
|
||||
// main site (hub → 4003) — NOT the scraper API domain. Use an explicit
|
||||
// env override (TERABOX_PUBLIC_BASE_URL), falling back to the scraper's
|
||||
// own public domain via site_url only when it's clearly the API host.
|
||||
let public_base = std::env::var("TERABOX_PUBLIC_BASE_URL").unwrap_or_else(|_| {
|
||||
let site = crate::config::CONFIG.urls.site_url.clone();
|
||||
if site.contains("scraper") || site.contains("api") {
|
||||
site
|
||||
} else {
|
||||
// default to the scraper API subdomain (this API's public host)
|
||||
"https://api.asepharyana.my.id".to_string()
|
||||
}
|
||||
});
|
||||
let proxy_dl = format!("{}/proxy/terabox?surl={}", public_base, surl);
|
||||
if let Some(download_url) = data.get("download_link").and_then(|v| v.as_str()) {
|
||||
let size_bytes = data.get("file_size").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||
|
||||
// PRIMARY = the cookie-authenticated public proxy. The raw terabox
|
||||
// dlink is only a secondary entry — it 403s without cookies, so it
|
||||
// must NOT be the first link users see.
|
||||
result.media.push(MediaItem {
|
||||
url: proxy_dl.clone(),
|
||||
quality: None,
|
||||
file_type: Some(MediaType::File),
|
||||
extension: None,
|
||||
thumbnail: None,
|
||||
file_size: Some(format_filesize(size_bytes)),
|
||||
size_bytes: Some(size_bytes),
|
||||
frame_width: None,
|
||||
frame_height: None,
|
||||
note: Some("Direct download via cookie-authenticated proxy (works).".into()),
|
||||
});
|
||||
result.media.push(MediaItem {
|
||||
url: download_url.to_string(),
|
||||
quality: None,
|
||||
file_type: Some(MediaType::File),
|
||||
extension: None,
|
||||
thumbnail: None,
|
||||
file_size: None,
|
||||
size_bytes: None,
|
||||
file_size: Some(format_filesize(size_bytes)),
|
||||
size_bytes: Some(size_bytes),
|
||||
frame_width: None,
|
||||
frame_height: None,
|
||||
note: None,
|
||||
note: Some("Raw CDN link (may require a TeraBox session cookie).".into()),
|
||||
});
|
||||
|
||||
Ok(result)
|
||||
} else {
|
||||
Ok(DownloadResult::error(
|
||||
resp.get("error")
|
||||
// Directory: add child file links
|
||||
if let Some(files) = data.get("files").and_then(|v| v.as_array()) {
|
||||
for f in files {
|
||||
if let Some(furl) = f.get("download_link").and_then(|v| v.as_str()) {
|
||||
let fsize = f.get("file_size").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||
result.media.push(MediaItem {
|
||||
url: furl.to_string(),
|
||||
quality: None,
|
||||
file_type: Some(MediaType::File),
|
||||
extension: None,
|
||||
thumbnail: None,
|
||||
file_size: Some(format_filesize(fsize)),
|
||||
size_bytes: Some(fsize),
|
||||
frame_width: None,
|
||||
frame_height: None,
|
||||
note: f
|
||||
.get("file_name")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("Failed to fetch TeraBox info"),
|
||||
))
|
||||
.map(|s| s.to_string()),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn fetch_videy(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||
let id = Url::parse(url)
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
pub mod ai;
|
||||
pub mod alqanime;
|
||||
pub mod downloader;
|
||||
pub mod image;
|
||||
@@ -10,8 +9,6 @@ pub mod proxy;
|
||||
pub mod search;
|
||||
pub mod stalk;
|
||||
pub mod tools;
|
||||
pub mod uploader;
|
||||
pub mod weebs;
|
||||
|
||||
pub use alqanime::AlqanimeRepository;
|
||||
pub use downloader::DownloaderRepository;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Infrastructure — Stalk utilities (github, youtube, instagram, tiktok, twitter).
|
||||
//! Infrastructure — Stalk utilities (github, youtube, twitter).
|
||||
//!
|
||||
//! Ported from Shirokami-API `scraper/stalk/*.js`.
|
||||
|
||||
@@ -179,7 +179,7 @@ pub async fn fetch_youtube_stalk(username: &str) -> Result<Value, String> {
|
||||
.and_then(|t| t.as_array())
|
||||
{
|
||||
if let Some(contents) = tabs
|
||||
.get(0)
|
||||
.first()
|
||||
.and_then(|t| t.pointer("/tabRenderer/content/sectionListRenderer/contents"))
|
||||
.and_then(|c| c.as_array())
|
||||
{
|
||||
@@ -223,8 +223,8 @@ fn extract_balanced_json(body: &str, open_brace: usize) -> Result<String, String
|
||||
let mut in_str = false;
|
||||
let mut escaped = false;
|
||||
let mut close = None;
|
||||
for i in open_brace..bytes.len() {
|
||||
let c = bytes[i] as char;
|
||||
for (i, b) in bytes.iter().enumerate().skip(open_brace) {
|
||||
let c = *b as char;
|
||||
if in_str {
|
||||
if escaped {
|
||||
escaped = false;
|
||||
@@ -341,7 +341,7 @@ fn video_from_lockup(lv: &Value) -> Value {
|
||||
.pointer("/metadata/lockupMetadataViewModel/metadata/contentMetadataViewModel/metadataRows/0/metadataParts")
|
||||
.and_then(|p| p.as_array())
|
||||
{
|
||||
if let Some(p0) = parts.get(0) {
|
||||
if let Some(p0) = parts.first() {
|
||||
view_count = p0
|
||||
.pointer("/text/content")
|
||||
.cloned()
|
||||
@@ -417,214 +417,6 @@ fn video_from_grid_video(gv: &Value) -> Value {
|
||||
v
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mobile Legends — gempaytopup.com stalk (CSRF token + POST)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub async fn fetch_ml_stalk(user_id: &str, zone_id: &str) -> Result<Value, String> {
|
||||
// 1. GET to obtain CSRF token + cookies
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.get("https://www.gempaytopup.com")
|
||||
.header(
|
||||
USER_AGENT,
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0 Safari/537.36",
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP: {}", e))?;
|
||||
let cookies = resp
|
||||
.headers()
|
||||
.get_all("set-cookie")
|
||||
.iter()
|
||||
.map(|v| {
|
||||
v.to_str()
|
||||
.unwrap_or("")
|
||||
.split(';')
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ");
|
||||
let body = resp.text().await.map_err(|e| format!("Body: {}", e))?;
|
||||
|
||||
let csrf_re = Regex::new(r#"<meta name="csrf-token" content="(.*?)">"#).unwrap();
|
||||
let csrf = csrf_re
|
||||
.captures(&body)
|
||||
.and_then(|c| c.get(1))
|
||||
.map(|m| m.as_str().to_string());
|
||||
|
||||
let (csrf, cookies) = match (csrf, cookies.is_empty()) {
|
||||
(Some(c), false) => (c, cookies),
|
||||
_ => return Ok(json!({"error": "Gagal mendapatkan CSRF token atau cookie."})),
|
||||
};
|
||||
|
||||
// 2. POST stalk-ml
|
||||
let payload = json!({"uid": user_id, "zone": zone_id});
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.post("https://www.gempaytopup.com/stalk-ml")
|
||||
.header("X-CSRF-Token", &csrf)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Cookie", &cookies)
|
||||
.json(&payload)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP: {}", e))?;
|
||||
let data: Value = resp.json().await.map_err(|e| format!("JSON: {}", e))?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Free Fire — freefirecommunity API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub async fn fetch_ff_stalk(user_id: &str) -> Result<Value, String> {
|
||||
let url = format!(
|
||||
"https://discordbot.freefirecommunity.com/player_info_api?uid={}®ion=id",
|
||||
user_id
|
||||
);
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.get(&url)
|
||||
.header("Origin", "https://www.freefirecommunity.com")
|
||||
.header(
|
||||
"Referer",
|
||||
"https://www.freefirecommunity.com/ff-account-info/",
|
||||
)
|
||||
.header(USER_AGENT, "Mozilla/5.0 (Linux; Android 10; K)")
|
||||
.header("Accept", "/")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP: {}", e))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("HTTP {}", resp.status()));
|
||||
}
|
||||
let data: Value = resp.json().await.map_err(|e| format!("JSON: {}", e))?;
|
||||
|
||||
let safe = |v: &Value| -> Value {
|
||||
if v.is_null() {
|
||||
Value::String("N/A".into())
|
||||
} else {
|
||||
v.clone()
|
||||
}
|
||||
};
|
||||
let arr_join = |v: &Value| -> Value {
|
||||
v.as_array()
|
||||
.map(|a| {
|
||||
Value::String(
|
||||
a.iter()
|
||||
.filter_map(|x| x.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", "),
|
||||
)
|
||||
})
|
||||
.unwrap_or(Value::String("-".into()))
|
||||
};
|
||||
let fmt_time = |v: &Value| -> Value {
|
||||
// unix timestamp -> YYYY-MM-DD (approx via chrono)
|
||||
v.as_i64()
|
||||
.map(|ts| {
|
||||
use chrono::{TimeZone, Utc};
|
||||
match Utc.timestamp_opt(ts, 0) {
|
||||
chrono::LocalResult::Single(dt) => {
|
||||
Value::String(dt.format("%Y-%m-%d %H:%M:%S").to_string())
|
||||
}
|
||||
_ => Value::String("N/A".into()),
|
||||
}
|
||||
})
|
||||
.unwrap_or(Value::String("N/A".into()))
|
||||
};
|
||||
|
||||
let d = &data["player_info"];
|
||||
let b = &d["basicInfo"];
|
||||
let c = &d["creditScoreInfo"];
|
||||
let p = &d["petInfo"];
|
||||
let prof = &d["profileInfo"];
|
||||
let s = &d["socialInfo"];
|
||||
|
||||
// battle tags
|
||||
let tags = s["battleTag"].as_array().cloned().unwrap_or_default();
|
||||
let tag_counts = s["battleTagCount"].as_array().cloned().unwrap_or_default();
|
||||
let battle_tags = if tags.is_empty() {
|
||||
Value::String("N/A".into())
|
||||
} else {
|
||||
Value::String(
|
||||
tags.iter()
|
||||
.enumerate()
|
||||
.map(|(i, t)| {
|
||||
let count = tag_counts.get(i).and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
format!("{} ({}x)", t.as_str().unwrap_or(""), count)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n"),
|
||||
)
|
||||
};
|
||||
|
||||
Ok(json!({
|
||||
"nickname": safe(&b["nickname"]),
|
||||
"accountId": safe(&b["accountId"]),
|
||||
"region": safe(&b["region"]),
|
||||
"level": safe(&b["level"]),
|
||||
"liked": safe(&b["liked"]),
|
||||
"rank": safe(&b["rank"]),
|
||||
"maxRank": safe(&b["maxRank"]),
|
||||
"csRank": safe(&b["csRank"]),
|
||||
"exp": safe(&b["exp"]),
|
||||
"createAt": fmt_time(&b["createAt"]),
|
||||
"lastLoginAt": fmt_time(&b["lastLoginAt"]),
|
||||
"rankingPoints": safe(&b["rankingPoints"]),
|
||||
"releaseVersion": safe(&b["releaseVersion"]),
|
||||
"seasonId": safe(&b["seasonId"]),
|
||||
"primeLevel": b["primeLevel"]["level"].is_null().then(|| Value::String("-".into())).unwrap_or_else(|| safe(&b["primeLevel"]["level"])),
|
||||
"diamondCost": safe(&d["diamondCostRes"]["diamondCost"]),
|
||||
"petName": safe(&p["name"]),
|
||||
"petLevel": safe(&p["level"]),
|
||||
"petExp": safe(&p["exp"]),
|
||||
"petSkinId": safe(&p["skinId"]),
|
||||
"petSkillId": safe(&p["selectedSkillId"]),
|
||||
"avatarId": safe(&prof["avatarId"]),
|
||||
"clothes": arr_join(&prof["clothes"]),
|
||||
"equipedSkills": arr_join(&prof["equipedSkills"]),
|
||||
"battleTags": battle_tags,
|
||||
"language": safe(&s["language"]),
|
||||
"rankShow": safe(&s["rankShow"]),
|
||||
"signature": safe(&s["signature"]),
|
||||
"creditScore": safe(&c["creditScore"]),
|
||||
"rewardState": safe(&c["rewardState"]),
|
||||
"bannerImage": format!("https://discordbot.freefirecommunity.com/banner_image_api?uid={}®ion=id", user_id),
|
||||
"outfitImage": format!("https://discordbot.freefirecommunity.com/outfit_image_api?uid={}®ion=id", user_id),
|
||||
}))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Genshin Impact — enka.network public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub async fn fetch_genshin_stalk(user_id: &str) -> Result<Value, String> {
|
||||
let url = format!("https://enka.network/u/{}/__data.json", user_id);
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.get(&url)
|
||||
.header(
|
||||
USER_AGENT,
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0 Safari/537.36",
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP: {}", e))?;
|
||||
if resp.status() == reqwest::StatusCode::NOT_FOUND {
|
||||
return Err("User tidak ditemukan".into());
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("HTTP {}", resp.status()));
|
||||
}
|
||||
let data: Value = resp.json().await.map_err(|e| format!("JSON: {}", e))?;
|
||||
// Pass through the raw JSON (Enka __data.json is already structured).
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Twitter — fxtwitter API
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -664,111 +456,3 @@ pub async fn fetch_twitter_stalk(username: &str) -> Result<Value, String> {
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TikTok — __UNIVERSAL_DATA_FOR_REHYDRATION__ JSON (V1 direct, no browser)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub async fn fetch_tiktok_stalk(username: &str) -> Result<Value, String> {
|
||||
let url = format!("https://www.tiktok.com/@{username}?_t=ZS-8tHANz7ieoS&_r=1");
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.get(&url)
|
||||
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP: {}", e))?;
|
||||
let html = resp.text().await.map_err(|e| format!("Body: {}", e))?;
|
||||
|
||||
// Extract __UNIVERSAL_DATA_FOR_REHYDRATION__ JSON
|
||||
let marker = r#"__UNIVERSAL_DATA_FOR_REHYDRATION__"#;
|
||||
let start = html.find(marker).ok_or("User tidak ditemukan")?;
|
||||
let json_start = html[start..]
|
||||
.find('>')
|
||||
.map(|i| start + i + 1)
|
||||
.ok_or("User tidak ditemukan")?;
|
||||
let json_end = html[json_start..]
|
||||
.find("</script>")
|
||||
.map(|i| json_start + i)
|
||||
.ok_or("User tidak ditemukan")?;
|
||||
let json_str = &html[json_start..json_end];
|
||||
|
||||
let parsed: Value = serde_json::from_str(json_str).map_err(|e| format!("JSON: {}", e))?;
|
||||
let user_info = parsed
|
||||
.pointer("/__DEFAULT_SCOPE__/webapp.user-detail/userInfo/user")
|
||||
.cloned()
|
||||
.ok_or("User tidak ditemukan")?;
|
||||
let stats = parsed
|
||||
.pointer("/__DEFAULT_SCOPE__/webapp.user-detail/userInfo/stats")
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null);
|
||||
|
||||
Ok(json!({
|
||||
"userInfo": {
|
||||
"id": user_info.get("id").cloned().unwrap_or(Value::Null),
|
||||
"username": user_info.get("uniqueId").cloned().unwrap_or(Value::Null),
|
||||
"name": user_info.get("nickname").cloned().unwrap_or(Value::Null),
|
||||
"avatar": user_info.get("avatarLarger").cloned().unwrap_or(Value::Null),
|
||||
"bio": user_info.get("signature").cloned().unwrap_or(Value::Null),
|
||||
"verified": user_info.get("verified").cloned().unwrap_or(Value::Bool(false)),
|
||||
"totalFollowers": stats.get("followerCount").cloned().unwrap_or(Value::from(0)),
|
||||
"totalFollowing": stats.get("followingCount").cloned().unwrap_or(Value::from(0)),
|
||||
"totalLikes": stats.get("heart").cloned().unwrap_or(Value::from(0)),
|
||||
"totalVideos": stats.get("videoCount").cloned().unwrap_or(Value::from(0)),
|
||||
"totalFriends": stats.get("friendCount").cloned().unwrap_or(Value::from(0)),
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Instagram — media.mollygram.com HTML scrape
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub async fn fetch_instagram_stalk(username: &str) -> Result<Value, String> {
|
||||
let url = format!("https://media.mollygram.com/?url={}", urlencode(username));
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.get(&url)
|
||||
.header("accept", "*/*")
|
||||
.header("origin", "https://mollygram.com")
|
||||
.header("referer", "https://mollygram.com/")
|
||||
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP: {}", e))?;
|
||||
let body = resp.text().await.map_err(|e| format!("Body: {}", e))?;
|
||||
|
||||
let re = |pat: &str| -> Option<String> {
|
||||
let rx = Regex::new(pat).ok()?;
|
||||
rx.captures(&body)
|
||||
.and_then(|c| c.get(1))
|
||||
.map(|m| m.as_str().trim().to_string())
|
||||
};
|
||||
|
||||
let avatar = re(r#"<img[^>]*class="[^"]*rounded-circle[^"]*"[^>]*src="([^"]+)""#)
|
||||
.or_else(|| re(r#"<img[^>]*src="([^"]+)"[^>]*class="[^"]*rounded-circle[^"]*""#));
|
||||
let uname = re(r#"<h4 class="mb-0">([^<]+)</h4>"#);
|
||||
let fullname = re(r#"<p class="text-muted">([^<]+)</p>"#);
|
||||
let bio = re(r#"<p class="text-dark"[^>]*>([^<]*)</p>"#);
|
||||
let posts = re(r#"<span class="d-block h5 mb-0">([^<]+)</span>\s*<div[^>]*>\s*posts\s*</div>"#);
|
||||
let followers =
|
||||
re(r#"<span class="d-block h5 mb-0">([^<]+)</span>\s*<div[^>]*>\s*followers\s*</div>"#);
|
||||
let following =
|
||||
re(r#"<span class="d-block h5 mb-0">([^<]+)</span>\s*<div[^>]*>\s*following\s*</div>"#);
|
||||
|
||||
Ok(json!({
|
||||
"avatar": avatar,
|
||||
"name": fullname.unwrap_or_default(),
|
||||
"username": uname.unwrap_or_else(|| username.to_string()),
|
||||
"posts": posts.unwrap_or_default(),
|
||||
"followers": followers.unwrap_or_default(),
|
||||
"following": following.unwrap_or_default(),
|
||||
"bio": bio.unwrap_or_default(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn urlencode(s: &str) -> String {
|
||||
s.replace('%', "%25")
|
||||
.replace('&', "%26")
|
||||
.replace('?', "%3F")
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
//! Infrastructure — Uploader utilities.
|
||||
//!
|
||||
//! The Shirokami-API source uploads to Ryzumi S3 (s3.ryzumi.vip) which is
|
||||
//! DNS-dead from this VPS. This implementation stores uploaded files to a
|
||||
//! local directory and serves them back, keeping the source's response shape:
|
||||
//! `{ success, url, fileName, size }`.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use fastrand;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
/// Directory where uploaded files are stored.
|
||||
const UPLOAD_DIR: &str = "/var/lib/scraper/uploads";
|
||||
|
||||
/// Base URL prefix used in the returned `url` field.
|
||||
fn public_base() -> String {
|
||||
"/uploader/file".to_string()
|
||||
}
|
||||
|
||||
/// Save an uploaded buffer to disk with a random filename + detected extension.
|
||||
pub fn save_file(buffer: &[u8], file_name: &str) -> Result<Value, String> {
|
||||
if buffer.is_empty() {
|
||||
return Err("Invalid content: buffer is required".into());
|
||||
}
|
||||
|
||||
let dir = PathBuf::from(UPLOAD_DIR);
|
||||
std::fs::create_dir_all(&dir).map_err(|e| format!("mkdir: {e}"))?;
|
||||
|
||||
// Detect extension: prefer the provided filename, else infer from magic.
|
||||
let extension = if file_name.contains('.') {
|
||||
file_name.split('.').last().unwrap_or("").to_lowercase()
|
||||
} else {
|
||||
infer_extension(buffer).to_string()
|
||||
};
|
||||
|
||||
if extension.is_empty() {
|
||||
return Err("Unable to determine file extension".into());
|
||||
}
|
||||
|
||||
let random = hex::encode(&fastrand::u64(..).to_le_bytes());
|
||||
let key = format!("{random}.{extension}");
|
||||
let path = dir.join(&key);
|
||||
|
||||
std::fs::write(&path, buffer).map_err(|e| format!("write: {e}"))?;
|
||||
|
||||
let url = format!("{}/{}", public_base(), key);
|
||||
Ok(json!({
|
||||
"success": true,
|
||||
"url": url,
|
||||
"fileName": key,
|
||||
"size": buffer.len(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Serve the file bytes for a given filename.
|
||||
pub fn read_file(file_name: &str) -> Result<(Vec<u8>, String), String> {
|
||||
// Prevent path traversal.
|
||||
let safe = Path::new(file_name)
|
||||
.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.ok_or("invalid filename")?;
|
||||
let path = PathBuf::from(UPLOAD_DIR).join(safe);
|
||||
let bytes = std::fs::read(&path).map_err(|e| format!("read: {e}"))?;
|
||||
let mime = infer_mime(safe).to_string();
|
||||
Ok((bytes, mime))
|
||||
}
|
||||
|
||||
fn infer_extension(buf: &[u8]) -> &'static str {
|
||||
if buf.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||
"png"
|
||||
} else if buf.starts_with(b"\xff\xd8\xff") {
|
||||
"jpg"
|
||||
} else if buf.starts_with(b"GIF87a") || buf.starts_with(b"GIF89a") {
|
||||
"gif"
|
||||
} else if buf.starts_with(b"RIFF") && &buf[8..12] == b"WEBP" {
|
||||
"webp"
|
||||
} else if buf.starts_with(b"%PDF") {
|
||||
"pdf"
|
||||
} else if buf.starts_with(b"PK\x03\x04") {
|
||||
"zip"
|
||||
} else {
|
||||
"bin"
|
||||
}
|
||||
}
|
||||
|
||||
fn infer_mime(name: &str) -> &'static str {
|
||||
let ext = name.rsplit('.').next().unwrap_or("");
|
||||
match ext {
|
||||
"png" => "image/png",
|
||||
"jpg" | "jpeg" => "image/jpeg",
|
||||
"gif" => "image/gif",
|
||||
"webp" => "image/webp",
|
||||
"pdf" => "application/pdf",
|
||||
"zip" => "application/zip",
|
||||
"txt" => "text/plain",
|
||||
"mp4" => "video/mp4",
|
||||
"webm" => "video/webm",
|
||||
"mp3" => "audio/mpeg",
|
||||
_ => "application/octet-stream",
|
||||
}
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
//! Infrastructure — Weebs endpoints (anime/manga info, waifu pics, whatanime).
|
||||
//!
|
||||
//! Ported from Shirokami-API `scraper/weebs/*.js`. These are public JSON
|
||||
//! APIs that we proxy/forward (jikan.moe, waifu.pics, trace.moe).
|
||||
|
||||
use crate::infrastructure::utils::http_client::http_client;
|
||||
use reqwest::header::USER_AGENT;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
/// Anime info from jikan.moe, ported from anime-search's `getAnimeInfo`.
|
||||
pub async fn fetch_anime_info(query: &str) -> Result<Value, String> {
|
||||
let url = format!("https://api.jikan.moe/v4/anime?q={}", urlencode(query));
|
||||
let data = get_json(&url).await?;
|
||||
|
||||
let arr = data
|
||||
.get("data")
|
||||
.and_then(|d| d.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
if arr.is_empty() {
|
||||
return Ok(json!({ "error": "Anime tidak ditemukan" }));
|
||||
}
|
||||
let anime = &arr[0];
|
||||
|
||||
let judul = anime
|
||||
.get("titles")
|
||||
.and_then(|t| t.as_array())
|
||||
.and_then(|t| t.first())
|
||||
.and_then(|t| t.get("title"))
|
||||
.and_then(|t| t.as_str())
|
||||
.unwrap_or("Title not available");
|
||||
let genrenya = join_names(anime.get("genres"));
|
||||
|
||||
Ok(json!({
|
||||
"title": judul,
|
||||
"url": anime.get("url").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"type": anime.get("type").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"score": anime.get("score").cloned().unwrap_or(Value::Null),
|
||||
"members": anime.get("members").cloned().unwrap_or(Value::Null),
|
||||
"status": anime.get("status").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"synopsis": anime.get("synopsis").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"favorites": anime.get("favorites").cloned().unwrap_or(Value::Null),
|
||||
"images": anime.get("images").cloned().unwrap_or(Value::Null),
|
||||
"genres": genrenya,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Manga info from jikan.moe, ported from anime-search's `getMangaInfo`.
|
||||
pub async fn fetch_manga_info(query: &str) -> Result<Value, String> {
|
||||
let url = format!("https://api.jikan.moe/v4/manga?q={}", urlencode(query));
|
||||
let data = get_json(&url).await?;
|
||||
|
||||
let arr = data
|
||||
.get("data")
|
||||
.and_then(|d| d.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
if arr.is_empty() {
|
||||
return Ok(json!({ "error": "Manga tidak ditemukan" }));
|
||||
}
|
||||
let manga = &arr[0];
|
||||
|
||||
let judul = manga
|
||||
.get("titles")
|
||||
.and_then(|t| t.as_array())
|
||||
.map(|tt| {
|
||||
tt.iter()
|
||||
.filter_map(|j| {
|
||||
let title = j.get("title").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let ty = j.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if title.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(format!("{} [{}]", title, ty))
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let genrenya = join_names(manga.get("genres"));
|
||||
|
||||
Ok(json!({
|
||||
"title": judul,
|
||||
"chapters": manga.get("chapters").cloned().unwrap_or(Value::Null),
|
||||
"type": manga.get("type").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"status": manga.get("status").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"genre": genrenya,
|
||||
"volumes": manga.get("volumes").cloned().unwrap_or(Value::Null),
|
||||
"favorites": manga.get("favorites").cloned().unwrap_or(Value::Null),
|
||||
"score": manga.get("score").cloned().unwrap_or(Value::Null),
|
||||
"scored": manga.get("scored").cloned().unwrap_or(Value::Null),
|
||||
"scored_by": manga.get("scored_by").cloned().unwrap_or(Value::Null),
|
||||
"rank": manga.get("rank").cloned().unwrap_or(Value::Null),
|
||||
"popularity": manga.get("popularity").cloned().unwrap_or(Value::Null),
|
||||
"members": manga.get("members").cloned().unwrap_or(Value::Null),
|
||||
"url": manga.get("url").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"background": manga.get("background").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"synopsis": manga.get("synopsis").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
}))
|
||||
}
|
||||
|
||||
/// What anime is in this screenshot? trace.moe reverse-image search.
|
||||
pub async fn fetch_whatanime(url: &str) -> Result<Value, String> {
|
||||
let api = format!(
|
||||
"https://api.trace.moe/search?cutBorders&url={}",
|
||||
urlencode(url)
|
||||
);
|
||||
let data = get_json(&api).await?;
|
||||
|
||||
let result = data
|
||||
.get("result")
|
||||
.and_then(|r| r.as_array())
|
||||
.and_then(|r| r.first())
|
||||
.cloned();
|
||||
let Some(r) = result else {
|
||||
return Ok(json!({ "error": "Anime tidak ditemukan" }));
|
||||
};
|
||||
|
||||
let similarity = r
|
||||
.get("similarity")
|
||||
.and_then(|s| s.as_f64())
|
||||
.map(|s| (s * 100.0).round() as i64)
|
||||
.unwrap_or(0);
|
||||
|
||||
Ok(json!({
|
||||
"judul": r.get("filename").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"episode": r.get("episode").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"similarity": similarity,
|
||||
"videoURL": r.get("video").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
"videoIMG": r.get("image").and_then(|v| v.as_str()).unwrap_or(""),
|
||||
}))
|
||||
}
|
||||
|
||||
/// SFW waifu image from waifu.pics.
|
||||
pub async fn fetch_sfw_waifu(tag: &str) -> Result<Value, String> {
|
||||
let url = format!("https://api.waifu.pics/sfw/{}", urlencode(tag));
|
||||
get_json(&url).await
|
||||
}
|
||||
|
||||
/// NSFW waifu image from waifu.pics.
|
||||
pub async fn fetch_nsfw_waifu(tag: &str) -> Result<Value, String> {
|
||||
let url = format!("https://api.waifu.pics/nsfw/{}", urlencode(tag));
|
||||
get_json(&url).await
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helpers
|
||||
// ============================================================================
|
||||
|
||||
async fn get_json(url: &str) -> Result<Value, String> {
|
||||
let resp = http_client()
|
||||
.client()
|
||||
.get(url)
|
||||
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("HTTP error: {}", e))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("HTTP {} from upstream", resp.status()));
|
||||
}
|
||||
resp.json::<Value>()
|
||||
.await
|
||||
.map_err(|e| format!("JSON parse: {}", e))
|
||||
}
|
||||
|
||||
/// Join an array of { name: "X" } objects into a comma-separated string.
|
||||
fn join_names(v: Option<&Value>) -> String {
|
||||
v.and_then(|g| g.as_array())
|
||||
.map(|arr| {
|
||||
arr.iter()
|
||||
.filter_map(|g| {
|
||||
g.get("name")
|
||||
.and_then(|n| n.as_str())
|
||||
.map(|s| s.to_string())
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Minimal URL-encode (space -> %20, keep alphanumerics and safe chars).
|
||||
fn urlencode(s: &str) -> String {
|
||||
let mut out = String::new();
|
||||
for b in s.bytes() {
|
||||
match b {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
||||
out.push(b as char)
|
||||
}
|
||||
b' ' => out.push_str("%20"),
|
||||
_ => out.push_str(&format!("%{:02X}", b)),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -66,6 +66,7 @@ use utoipa::OpenApi;
|
||||
crate::presentation::handler::downloader::download_streamable,
|
||||
crate::presentation::handler::downloader::download_videy,
|
||||
crate::presentation::handler::downloader::download_bilibili,
|
||||
crate::presentation::handler::downloader::serve_merged_file,
|
||||
// ── Misc module ────────────────────────────────────────────
|
||||
crate::presentation::handler::misc::currency_converter_handler,
|
||||
crate::presentation::handler::misc::harga_emas_handler,
|
||||
@@ -74,18 +75,7 @@ use utoipa::OpenApi;
|
||||
// ── Stalk module ───────────────────────────────────────────
|
||||
crate::presentation::handler::stalk::github_handler,
|
||||
crate::presentation::handler::stalk::youtube_handler,
|
||||
crate::presentation::handler::stalk::mobile_legends_handler,
|
||||
crate::presentation::handler::stalk::free_fire_handler,
|
||||
crate::presentation::handler::stalk::genshin_handler,
|
||||
crate::presentation::handler::stalk::twitter_handler,
|
||||
crate::presentation::handler::stalk::tiktok_handler,
|
||||
crate::presentation::handler::stalk::instagram_handler,
|
||||
// ── Weebs module ───────────────────────────────────────────
|
||||
crate::presentation::handler::weebs::anime_info_handler,
|
||||
crate::presentation::handler::weebs::manga_info_handler,
|
||||
crate::presentation::handler::weebs::whatanime_handler,
|
||||
crate::presentation::handler::weebs::sfw_waifu_handler,
|
||||
crate::presentation::handler::weebs::nsfw_waifu_handler,
|
||||
// ── Search module ──────────────────────────────────────────
|
||||
crate::presentation::handler::search::bmkg_handler,
|
||||
crate::presentation::handler::search::jadwal_sholat_handler,
|
||||
@@ -102,13 +92,6 @@ use utoipa::OpenApi;
|
||||
// ── Image module ───────────────────────────────────────────
|
||||
crate::presentation::handler::image::brat_handler,
|
||||
crate::presentation::handler::image::brat_animated_handler,
|
||||
// ── Uploader module ────────────────────────────────────────
|
||||
crate::presentation::handler::uploader::ryzencdn_handler,
|
||||
crate::presentation::handler::uploader::serve_file_handler,
|
||||
// ── AI module ──────────────────────────────────────────────
|
||||
crate::presentation::handler::ai::chat_handler,
|
||||
crate::presentation::handler::ai::gemini_handler,
|
||||
crate::presentation::handler::ai::image_handler,
|
||||
),
|
||||
components(
|
||||
schemas(
|
||||
@@ -122,13 +105,10 @@ use utoipa::OpenApi;
|
||||
(name = "komik", description = "Komik endpoints"),
|
||||
(name = "download", description = "Media download endpoints (23 platforms)"),
|
||||
(name = "misc", description = "Miscellaneous utilities (currency, gold, server)"),
|
||||
(name = "stalk", description = "User profile stalking (GitHub, YT, Twitter, TikTok, IG, FF, ML, Genshin)"),
|
||||
(name = "weebs", description = "Weeb endpoints (anime info, manga, anime search, waifu)"),
|
||||
(name = "stalk", description = "User profile stalking (GitHub, YT, Twitter)"),
|
||||
(name = "search", description = "Search endpoints (BMKG, prayer times, weather, YouTube, Google)"),
|
||||
(name = "tool", description = "Utility tools (WHOIS, IP lookup, URL shortener, tracking, hosting)"),
|
||||
(name = "image", description = "Image generation (brat static & animated)"),
|
||||
(name = "uploader", description = "File upload and serve"),
|
||||
(name = "ai", description = "AI endpoints (chat, gemini, image generation)"),
|
||||
)
|
||||
)]
|
||||
pub struct ModuleApiDoc;
|
||||
@@ -1,103 +0,0 @@
|
||||
//! Axum handlers for AI endpoints (Pollinations).
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::Query;
|
||||
use axum::http::{header, StatusCode};
|
||||
use axum::response::Response;
|
||||
use axum::Json;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::application::ai as use_cases;
|
||||
use crate::presentation::error::AppError;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct AiParams {
|
||||
pub text: Option<String>,
|
||||
pub prompt: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub image_url: Option<String>,
|
||||
pub url: Option<String>,
|
||||
pub width: Option<u32>,
|
||||
pub height: Option<u32>,
|
||||
}
|
||||
|
||||
/// GET /ai/chat?text=...&prompt=...&model=...
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/ai/chat",
|
||||
tag = "ai",
|
||||
operation_id = "ai_chat_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn chat_handler(Query(params): Query<AiParams>) -> Result<Json<Value>, AppError> {
|
||||
let text = params
|
||||
.text
|
||||
.ok_or_else(|| AppError::BadRequest("Missing 'text' parameter".to_string()))?;
|
||||
let result = use_cases::chat(text, params.prompt.unwrap_or_default(), params.model).await?;
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
/// GET /ai/gemini?text=...&prompt=...&image_url=...
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/ai/gemini",
|
||||
tag = "ai",
|
||||
operation_id = "ai_gemini_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn gemini_handler(Query(params): Query<AiParams>) -> Result<Json<Value>, AppError> {
|
||||
let text = params
|
||||
.text
|
||||
.ok_or_else(|| AppError::BadRequest("Missing 'text' parameter".to_string()))?;
|
||||
let result =
|
||||
use_cases::gemini(text, params.prompt.unwrap_or_default(), params.image_url).await?;
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
/// GET /ai/image?prompt=...&model=...&width=...&height=... — returns image bytes.
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/ai/image",
|
||||
tag = "ai",
|
||||
operation_id = "ai_image_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn image_handler(Query(params): Query<AiParams>) -> Result<Response<Body>, AppError> {
|
||||
let prompt = params
|
||||
.prompt
|
||||
.ok_or_else(|| AppError::BadRequest("Missing 'prompt' parameter".to_string()))?;
|
||||
let model = params.model.unwrap_or_else(|| "flux".to_string());
|
||||
let width = params.width.unwrap_or(1024);
|
||||
let height = params.height.unwrap_or(1024);
|
||||
|
||||
let bytes = use_cases::image(prompt, model, width, height).await?;
|
||||
// Pollinations flux returns JPEG by default.
|
||||
let mime = if bytes.starts_with(b"\xff\xd8\xff") {
|
||||
"image/jpeg"
|
||||
} else if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
|
||||
"image/png"
|
||||
} else {
|
||||
"application/octet-stream"
|
||||
};
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, mime)
|
||||
.body(Body::from(bytes))
|
||||
.map_err(|e| AppError::Internal(format!("build response: {e}")))
|
||||
}
|
||||
@@ -21,6 +21,9 @@ pub struct DownloadParams {
|
||||
pub cookies: Option<String>,
|
||||
pub api_key: Option<String>,
|
||||
pub quality: Option<String>,
|
||||
/// When `true`, YouTube downloads are merged server-side (video + audio
|
||||
/// into a single MP4) instead of returning separate video/audio URLs.
|
||||
pub merge: Option<String>,
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
@@ -44,7 +47,25 @@ pub struct DownloadParams {
|
||||
pub async fn download(
|
||||
Query(params): Query<DownloadParams>,
|
||||
) -> Result<Json<DownloadResponse>, AppError> {
|
||||
let result = use_cases::download_all_in_one(¶ms.url, params.cookies.as_deref()).await?;
|
||||
let merge = params
|
||||
.merge
|
||||
.as_deref()
|
||||
.map(|v| v == "true" || v == "1")
|
||||
.unwrap_or(false);
|
||||
let result = if merge {
|
||||
// All-in-one with merge=true → force merge for YouTube URLs.
|
||||
if crate::application::downloader::detect_platform(¶ms.url) == "youtube" {
|
||||
use_cases::download_youtube_merge(
|
||||
¶ms.url,
|
||||
params.quality.as_deref().unwrap_or("720"),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
use_cases::download_all_in_one(¶ms.url, params.cookies.as_deref()).await?
|
||||
}
|
||||
} else {
|
||||
use_cases::download_all_in_one(¶ms.url, params.cookies.as_deref()).await?
|
||||
};
|
||||
Ok(Json(DownloadResponse::ok(result)))
|
||||
}
|
||||
|
||||
@@ -152,9 +173,17 @@ pub async fn download_tiktok(
|
||||
pub async fn download_youtube(
|
||||
Query(params): Query<DownloadParams>,
|
||||
) -> Result<Json<DownloadResponse>, AppError> {
|
||||
let result =
|
||||
use_cases::download_youtube(¶ms.url, params.quality.as_deref().unwrap_or("720"))
|
||||
.await?;
|
||||
let quality = params.quality.as_deref().unwrap_or("720");
|
||||
let merge = params
|
||||
.merge
|
||||
.as_deref()
|
||||
.map(|v| v == "true" || v == "1")
|
||||
.unwrap_or(false);
|
||||
let result = if merge {
|
||||
use_cases::download_youtube_merge(¶ms.url, quality).await?
|
||||
} else {
|
||||
use_cases::download_youtube(¶ms.url, quality).await?
|
||||
};
|
||||
Ok(Json(DownloadResponse::ok(result)))
|
||||
}
|
||||
|
||||
@@ -284,6 +313,101 @@ pub async fn download_terabox(
|
||||
Ok(Json(DownloadResponse::ok(result)))
|
||||
}
|
||||
|
||||
/// Stream a TeraBox file through this API.
|
||||
/// `/proxy/terabox?surl=...`
|
||||
///
|
||||
/// The resolver's `/download` endpoint replays the full 302 chain WITH
|
||||
/// cookies and streams real bytes — this handler forwards those bytes to the
|
||||
/// caller so the returned URL is a public, downloadable link (not a 403
|
||||
/// dlink, not a private 127.0.0.1 URL).
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/proxy/terabox",
|
||||
tag = "download",
|
||||
operation_id = "dl_proxy_terabox",
|
||||
params(
|
||||
("surl" = String, Query, description = "TeraBox share short URL ID"),
|
||||
("filename" = Option<String>, Query, description = "Download file name hint"),
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "File stream"),
|
||||
(status = 400, description = "Missing surl"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
pub async fn proxy_terabox(
|
||||
Query(params): Query<TeraboxProxyParams>,
|
||||
) -> Result<axum::response::Response, AppError> {
|
||||
let surl = params.surl;
|
||||
if surl.is_empty() {
|
||||
return Err(AppError::BadRequest("missing surl parameter".into()));
|
||||
}
|
||||
|
||||
let resolver_url = std::env::var("TERABOX_RESOLVER_URL")
|
||||
.unwrap_or_else(|_| "http://127.0.0.1:4092".to_string());
|
||||
let fetch_url = format!("{}/download?surl={}", resolver_url, surl);
|
||||
|
||||
// Ask the resolver for the redirect chain (it follows 302s internally with
|
||||
// cookies) — but we only want the final CDN stream URL. The resolver
|
||||
// already streams; simplest reliable path is to just proxy the resolver's
|
||||
// response body as-is.
|
||||
let client = crate::infrastructure::utils::http_client::http_client();
|
||||
|
||||
let resp = client
|
||||
.client()
|
||||
.get(&fetch_url)
|
||||
.header("User-Agent", "Mozilla/5.0")
|
||||
.timeout(std::time::Duration::from_secs(120))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| AppError::ScraperError(format!("terabox proxy fetch failed: {e}")))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(AppError::ScraperError(format!(
|
||||
"terabox proxy returned HTTP {status}: {body}"
|
||||
)));
|
||||
}
|
||||
|
||||
let content_type = resp
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("application/octet-stream")
|
||||
.to_string();
|
||||
|
||||
let content_disposition = resp
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_DISPOSITION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_else(|| {
|
||||
let fname = params
|
||||
.filename
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| "terabox-download.bin".to_string());
|
||||
format!("attachment; filename=\"{fname}\"")
|
||||
});
|
||||
|
||||
let stream = resp.bytes_stream();
|
||||
|
||||
Ok(axum::response::Response::builder()
|
||||
.header("Content-Type", content_type)
|
||||
.header("Content-Disposition", content_disposition)
|
||||
.header("X-Accel-Buffering", "no")
|
||||
.header("Cache-Control", "no-store")
|
||||
.body(axum::body::Body::from_stream(stream))
|
||||
.map_err(|e| AppError::Internal(format!("build stream response: {e}")))?)
|
||||
}
|
||||
|
||||
/// Params for the TeraBox streaming proxy.
|
||||
#[derive(Debug, Deserialize, IntoParams)]
|
||||
pub struct TeraboxProxyParams {
|
||||
pub surl: String,
|
||||
pub filename: Option<String>,
|
||||
}
|
||||
|
||||
/// Download from Google Drive.
|
||||
/// `/download/gdrive?url=...`
|
||||
#[utoipa::path(
|
||||
@@ -556,3 +680,62 @@ pub async fn download_bilibili(
|
||||
let result = use_cases::download_bilibili(¶ms.url).await?;
|
||||
Ok(Json(DownloadResponse::ok(result)))
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Merged-file static serving
|
||||
// ========================================================================
|
||||
|
||||
/// Serve a merged YouTube MP4 produced by `/download/youtube?merge=true`.
|
||||
/// Path is sanitised: only a plain filename (no `/`, `..`) inside the
|
||||
/// `yt_merge` uploads dir is allowed.
|
||||
///
|
||||
/// `/file/yt_merge/{filename}`
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/file/yt_merge/{filename}",
|
||||
tag = "download",
|
||||
operation_id = "dl_serve_merged_file",
|
||||
params(
|
||||
("filename" = String, Path, description = "Merged MP4 file name")
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "Merged MP4 file", content_type = "video/mp4"),
|
||||
(status = 404, description = "File not found or invalid name")
|
||||
)
|
||||
)]
|
||||
pub async fn serve_merged_file(
|
||||
axum::extract::Path(filename): axum::extract::Path<String>,
|
||||
) -> Result<axum::response::Response, AppError> {
|
||||
// Reject any path traversal or nested segments.
|
||||
if filename.contains('/')
|
||||
|| filename.contains('\\')
|
||||
|| filename.contains("..")
|
||||
|| filename.is_empty()
|
||||
{
|
||||
return Err(AppError::NotFound(format!("invalid file name: {filename}")));
|
||||
}
|
||||
|
||||
let dir = std::path::PathBuf::from("/var/lib/scraper/uploads/yt_merge");
|
||||
let path = dir.join(&filename);
|
||||
let bytes = tokio::fs::read(&path)
|
||||
.await
|
||||
.map_err(|_| AppError::NotFound(format!("file not found: {filename}")))?;
|
||||
|
||||
let content_type = if filename.ends_with(".mp4") {
|
||||
"video/mp4"
|
||||
} else if filename.ends_with(".webm") {
|
||||
"video/webm"
|
||||
} else {
|
||||
"application/octet-stream"
|
||||
};
|
||||
|
||||
Ok(axum::response::Response::builder()
|
||||
.header("Content-Type", content_type)
|
||||
.header("Content-Length", bytes.len().to_string())
|
||||
.header(
|
||||
"Content-Disposition",
|
||||
format!("attachment; filename=\"{filename}\""),
|
||||
)
|
||||
.body(axum::body::Body::from(bytes))
|
||||
.map_err(|e| AppError::Internal(format!("build response: {e}")))?)
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
pub mod ai;
|
||||
pub mod anime;
|
||||
pub mod anime2;
|
||||
pub mod downloader;
|
||||
@@ -10,5 +9,3 @@ pub mod proxy;
|
||||
pub mod search;
|
||||
pub mod stalk;
|
||||
pub mod tools;
|
||||
pub mod uploader;
|
||||
pub mod weebs;
|
||||
@@ -68,82 +68,6 @@ pub async fn youtube_handler(
|
||||
Ok((StatusCode::OK, Json(result)))
|
||||
}
|
||||
|
||||
/// GET /stalk/mobile-legends?user_id=...&zone_id=... (aliases: userid, zoneid)
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/stalk/mobile-legends",
|
||||
tag = "stalk",
|
||||
operation_id = "stalk_mobile_legends",
|
||||
params(StalkParams),
|
||||
responses(
|
||||
(status = 200, description = "Mobile Legends player info", body = Value),
|
||||
(status = 400, description = "Missing user_id/zone_id"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
pub async fn mobile_legends_handler(
|
||||
Query(params): Query<StalkParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let user_id = params
|
||||
.user_id
|
||||
.or(params.userid)
|
||||
.ok_or_else(|| AppError::BadRequest("Missing 'user_id' parameter".to_string()))?;
|
||||
let zone_id = params
|
||||
.zone_id
|
||||
.or(params.zoneid)
|
||||
.ok_or_else(|| AppError::BadRequest("Missing 'zone_id' parameter".to_string()))?;
|
||||
let result = use_cases::mobile_legends(&user_id, &zone_id).await?;
|
||||
Ok((StatusCode::OK, Json(result)))
|
||||
}
|
||||
|
||||
/// GET /stalk/free-fire?user_id=... (alias: userid)
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/stalk/free-fire",
|
||||
tag = "stalk",
|
||||
operation_id = "stalk_free_fire",
|
||||
params(StalkParams),
|
||||
responses(
|
||||
(status = 200, description = "Free Fire player info", body = Value),
|
||||
(status = 400, description = "Missing user_id"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
pub async fn free_fire_handler(
|
||||
Query(params): Query<StalkParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let user_id = params
|
||||
.user_id
|
||||
.or(params.userid)
|
||||
.ok_or_else(|| AppError::BadRequest("Missing 'user_id' parameter".to_string()))?;
|
||||
let result = use_cases::free_fire(&user_id).await?;
|
||||
Ok((StatusCode::OK, Json(result)))
|
||||
}
|
||||
|
||||
/// GET /stalk/genshin-impact?user_id=... (alias: userid)
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/stalk/genshin-impact",
|
||||
tag = "stalk",
|
||||
operation_id = "stalk_genshin",
|
||||
params(StalkParams),
|
||||
responses(
|
||||
(status = 200, description = "Genshin Impact profile", body = Value),
|
||||
(status = 400, description = "Missing user_id"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
pub async fn genshin_handler(
|
||||
Query(params): Query<StalkParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let user_id = params
|
||||
.user_id
|
||||
.or(params.userid)
|
||||
.ok_or_else(|| AppError::BadRequest("Missing 'user_id' parameter".to_string()))?;
|
||||
let result = use_cases::genshin(&user_id).await?;
|
||||
Ok((StatusCode::OK, Json(result)))
|
||||
}
|
||||
|
||||
/// GET /stalk/twitter?username=...
|
||||
#[utoipa::path(
|
||||
get,
|
||||
@@ -166,49 +90,3 @@ pub async fn twitter_handler(
|
||||
let result = use_cases::twitter(&username).await?;
|
||||
Ok((StatusCode::OK, Json(result)))
|
||||
}
|
||||
|
||||
/// GET /stalk/tiktok?username=...
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/stalk/tiktok",
|
||||
tag = "stalk",
|
||||
operation_id = "stalk_tiktok",
|
||||
params(StalkParams),
|
||||
responses(
|
||||
(status = 200, description = "TikTok user profile", body = Value),
|
||||
(status = 400, description = "Missing username"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
pub async fn tiktok_handler(
|
||||
Query(params): Query<StalkParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let username = params
|
||||
.username
|
||||
.ok_or_else(|| AppError::BadRequest("Missing 'username' parameter".to_string()))?;
|
||||
let result = use_cases::tiktok(&username).await?;
|
||||
Ok((StatusCode::OK, Json(result)))
|
||||
}
|
||||
|
||||
/// GET /stalk/instagram?username=...
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/stalk/instagram",
|
||||
tag = "stalk",
|
||||
operation_id = "stalk_instagram",
|
||||
params(StalkParams),
|
||||
responses(
|
||||
(status = 200, description = "Instagram user profile", body = Value),
|
||||
(status = 400, description = "Missing username"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
pub async fn instagram_handler(
|
||||
Query(params): Query<StalkParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let username = params
|
||||
.username
|
||||
.ok_or_else(|| AppError::BadRequest("Missing 'username' parameter".to_string()))?;
|
||||
let result = use_cases::instagram(&username).await?;
|
||||
Ok((StatusCode::OK, Json(result)))
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
//! Axum handlers for the uploader.
|
||||
//!
|
||||
//! POST /uploader/ryzencdn — multipart file upload, saved to local disk.
|
||||
//! GET /uploader/file/:name — serve an uploaded file.
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{Multipart, Path};
|
||||
use axum::http::{header, StatusCode};
|
||||
use axum::response::Response;
|
||||
use axum::Json;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::application::uploader as use_cases;
|
||||
use crate::presentation::error::AppError;
|
||||
|
||||
/// POST /uploader/ryzencdn — accept a multipart file upload.
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/uploader/ryzencdn",
|
||||
tag = "uploader",
|
||||
operation_id = "uploader_ryzencdn_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn ryzencdn_handler(mut multipart: Multipart) -> Result<Json<Value>, AppError> {
|
||||
// Read the first file field.
|
||||
let mut file_name = String::from("");
|
||||
let mut data: Vec<u8> = Vec::new();
|
||||
|
||||
while let Some(field) = multipart
|
||||
.next_field()
|
||||
.await
|
||||
.map_err(|e| AppError::BadRequest(format!("multipart: {e}")))?
|
||||
{
|
||||
let name = field.name().unwrap_or("file").to_string();
|
||||
if name == "file" {
|
||||
file_name = field.file_name().unwrap_or("").to_string();
|
||||
let content = field
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| AppError::BadRequest(format!("bytes: {e}")))?;
|
||||
data = content.to_vec();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if data.is_empty() {
|
||||
return Err(AppError::BadRequest(
|
||||
"No file field named 'file' in multipart body".into(),
|
||||
));
|
||||
}
|
||||
|
||||
let result = use_cases::upload(&data, &file_name)?;
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
/// GET /uploader/file/:name — serve an uploaded file.
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/uploader/file/:name",
|
||||
tag = "uploader",
|
||||
operation_id = "uploader_serve_file_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn serve_file_handler(Path(name): Path<String>) -> Result<Response<Body>, AppError> {
|
||||
let (bytes, mime) = use_cases::serve(&name)?;
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, mime)
|
||||
.header(header::CACHE_CONTROL, "public, max-age=86400")
|
||||
.body(Body::from(bytes))
|
||||
.map_err(|e| AppError::Internal(format!("build response: {e}")))
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
//! Axum handlers for weebs endpoints.
|
||||
//!
|
||||
//! Ported from Shirokami-API `scraper/weebs/*.js`.
|
||||
|
||||
use axum::extract::Query;
|
||||
use axum::http::StatusCode;
|
||||
use axum::Json;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::application::weebs as use_cases;
|
||||
use crate::presentation::error::AppError;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct WeebsParams {
|
||||
pub query: Option<String>,
|
||||
pub q: Option<String>,
|
||||
pub url: Option<String>,
|
||||
pub tag: Option<String>,
|
||||
}
|
||||
|
||||
fn require(v: Option<&String>, name: &str) -> Result<String, AppError> {
|
||||
v.cloned()
|
||||
.ok_or_else(|| AppError::BadRequest(format!("Missing '{}' parameter", name)))
|
||||
}
|
||||
|
||||
/// GET /weebs/anime-info?query=naruto
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/weebs/anime-info",
|
||||
tag = "weebs",
|
||||
operation_id = "weebs_anime_info_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn anime_info_handler(
|
||||
Query(p): Query<WeebsParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let query = require(p.query.as_ref().or(p.q.as_ref()), "query")?;
|
||||
Ok((StatusCode::OK, Json(use_cases::anime_info(&query).await?)))
|
||||
}
|
||||
|
||||
/// GET /weebs/manga-info?query=one%20piece
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/weebs/manga-info",
|
||||
tag = "weebs",
|
||||
operation_id = "weebs_manga_info_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn manga_info_handler(
|
||||
Query(p): Query<WeebsParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let query = require(p.query.as_ref().or(p.q.as_ref()), "query")?;
|
||||
Ok((StatusCode::OK, Json(use_cases::manga_info(&query).await?)))
|
||||
}
|
||||
|
||||
/// GET /weebs/whatanime?url=<image-url>
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/weebs/whatanime",
|
||||
tag = "weebs",
|
||||
operation_id = "weebs_whatanime_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn whatanime_handler(
|
||||
Query(p): Query<WeebsParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let url = require(p.url.as_ref(), "url")?;
|
||||
Ok((StatusCode::OK, Json(use_cases::whatanime(&url).await?)))
|
||||
}
|
||||
|
||||
/// GET /weebs/sfw-waifu?tag=waifu
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/weebs/sfw-waifu",
|
||||
tag = "weebs",
|
||||
operation_id = "weebs_sfw_waifu_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn sfw_waifu_handler(
|
||||
Query(p): Query<WeebsParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let tag = require(p.tag.as_ref(), "tag")?;
|
||||
Ok((StatusCode::OK, Json(use_cases::sfw_waifu(&tag).await?)))
|
||||
}
|
||||
|
||||
/// GET /weebs/nsfw-waifu?tag=waifu
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/weebs/nsfw-waifu",
|
||||
tag = "weebs",
|
||||
operation_id = "weebs_nsfw_waifu_handler",
|
||||
responses(
|
||||
(status = 200, description = "OK", body = serde_json::Value),
|
||||
(status = 400, description = "Bad Request"),
|
||||
(status = 502, description = "Upstream error"),
|
||||
)
|
||||
)]
|
||||
|
||||
pub async fn nsfw_waifu_handler(
|
||||
Query(p): Query<WeebsParams>,
|
||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||
let tag = require(p.tag.as_ref(), "tag")?;
|
||||
Ok((StatusCode::OK, Json(use_cases::nsfw_waifu(&tag).await?)))
|
||||
}
|
||||
@@ -184,6 +184,10 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
||||
"/download/youtube/mp3",
|
||||
axum::routing::get(crate::presentation::handler::downloader::download_youtube_mp3),
|
||||
)
|
||||
.route(
|
||||
"/file/yt_merge/{filename}",
|
||||
axum::routing::get(crate::presentation::handler::downloader::serve_merged_file),
|
||||
)
|
||||
.route(
|
||||
"/download/spotify",
|
||||
axum::routing::get(crate::presentation::handler::downloader::download_spotify),
|
||||
@@ -204,6 +208,10 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
||||
"/download/terabox",
|
||||
axum::routing::get(crate::presentation::handler::downloader::download_terabox),
|
||||
)
|
||||
.route(
|
||||
"/proxy/terabox",
|
||||
axum::routing::get(crate::presentation::handler::downloader::proxy_terabox),
|
||||
)
|
||||
.route(
|
||||
"/download/gdrive",
|
||||
axum::routing::get(crate::presentation::handler::downloader::download_gdrive),
|
||||
@@ -282,51 +290,10 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
||||
"/stalk/youtube",
|
||||
axum::routing::get(crate::presentation::handler::stalk::youtube_handler),
|
||||
)
|
||||
.route(
|
||||
"/stalk/mobile-legends",
|
||||
axum::routing::get(crate::presentation::handler::stalk::mobile_legends_handler),
|
||||
)
|
||||
.route(
|
||||
"/stalk/free-fire",
|
||||
axum::routing::get(crate::presentation::handler::stalk::free_fire_handler),
|
||||
)
|
||||
.route(
|
||||
"/stalk/genshin-impact",
|
||||
axum::routing::get(crate::presentation::handler::stalk::genshin_handler),
|
||||
)
|
||||
.route(
|
||||
"/stalk/twitter",
|
||||
axum::routing::get(crate::presentation::handler::stalk::twitter_handler),
|
||||
)
|
||||
.route(
|
||||
"/stalk/tiktok",
|
||||
axum::routing::get(crate::presentation::handler::stalk::tiktok_handler),
|
||||
)
|
||||
.route(
|
||||
"/stalk/instagram",
|
||||
axum::routing::get(crate::presentation::handler::stalk::instagram_handler),
|
||||
)
|
||||
// Weebs routes
|
||||
.route(
|
||||
"/weebs/anime-info",
|
||||
axum::routing::get(crate::presentation::handler::weebs::anime_info_handler),
|
||||
)
|
||||
.route(
|
||||
"/weebs/manga-info",
|
||||
axum::routing::get(crate::presentation::handler::weebs::manga_info_handler),
|
||||
)
|
||||
.route(
|
||||
"/weebs/whatanime",
|
||||
axum::routing::get(crate::presentation::handler::weebs::whatanime_handler),
|
||||
)
|
||||
.route(
|
||||
"/weebs/sfw-waifu",
|
||||
axum::routing::get(crate::presentation::handler::weebs::sfw_waifu_handler),
|
||||
)
|
||||
.route(
|
||||
"/weebs/nsfw-waifu",
|
||||
axum::routing::get(crate::presentation::handler::weebs::nsfw_waifu_handler),
|
||||
)
|
||||
// Search routes
|
||||
.route(
|
||||
"/search/bmkg",
|
||||
@@ -382,28 +349,6 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
||||
"/image/brat/animated",
|
||||
axum::routing::get(crate::presentation::handler::image::brat_animated_handler),
|
||||
)
|
||||
// Uploader routes
|
||||
.route(
|
||||
"/uploader/ryzencdn",
|
||||
axum::routing::post(crate::presentation::handler::uploader::ryzencdn_handler),
|
||||
)
|
||||
.route(
|
||||
"/uploader/file/{name}",
|
||||
axum::routing::get(crate::presentation::handler::uploader::serve_file_handler),
|
||||
)
|
||||
// AI routes
|
||||
.route(
|
||||
"/ai/chat",
|
||||
axum::routing::get(crate::presentation::handler::ai::chat_handler),
|
||||
)
|
||||
.route(
|
||||
"/ai/gemini",
|
||||
axum::routing::get(crate::presentation::handler::ai::gemini_handler),
|
||||
)
|
||||
.route(
|
||||
"/ai/image",
|
||||
axum::routing::get(crate::presentation::handler::ai::image_handler),
|
||||
)
|
||||
// Health
|
||||
.route(
|
||||
"/health",
|
||||
|
||||
Reference in new issue
Block a user