Compare commits
16
Commits
091a68bd55
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7d68ff5620 | ||
|
|
8b9bdeb56a | ||
|
|
029837d0f1 | ||
|
|
8a55fd91a4 | ||
|
|
5cec284cd6 | ||
|
|
adfbbfcea7 | ||
|
|
09c706cc61 | ||
|
|
93828c7253 | ||
|
|
5470d74921 | ||
|
|
bbb8bed389 | ||
|
|
e6f543078b | ||
|
|
2769b11918 | ||
|
|
8d74d04b43 | ||
|
|
4efcf1f0ea | ||
|
|
278d4fe32a | ||
|
|
7ffc1bdd81 |
No files matched your search
@@ -15,7 +15,7 @@
|
|||||||
};
|
};
|
||||||
|
|
||||||
scraper = pkgs.stdenv.mkDerivation {
|
scraper = pkgs.stdenv.mkDerivation {
|
||||||
name = "scraper-0.1.0";
|
name = "scraper-0.1.1";
|
||||||
src = ./.;
|
src = ./.;
|
||||||
|
|
||||||
nativeBuildInputs = with pkgs; [
|
nativeBuildInputs = with pkgs; [
|
||||||
|
|||||||
@@ -42,6 +42,13 @@ pub async fn download_youtube(url: &str, quality: &str) -> Result<DownloadResult
|
|||||||
DownloaderRepository::download_youtube(url, quality).await
|
DownloaderRepository::download_youtube(url, quality).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn download_youtube_merge(
|
||||||
|
url: &str,
|
||||||
|
quality: &str,
|
||||||
|
) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
DownloaderRepository::fetch_youtube_merge(url, quality).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn download_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
|
pub async fn download_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
DownloaderRepository::download_youtube_mp3(url).await
|
DownloaderRepository::download_youtube_mp3(url).await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
//! Application use-cases for image generation.
|
||||||
|
|
||||||
|
use crate::domain::error::ScrapingError;
|
||||||
|
use crate::infrastructure::repository::image as Repo;
|
||||||
|
|
||||||
|
pub async fn brat(text: &str) -> Result<Vec<u8>, ScrapingError> {
|
||||||
|
Repo::fetch_brat(text).await.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn brat_animated(text: &str) -> Result<Vec<u8>, ScrapingError> {
|
||||||
|
Repo::fetch_brat_animated(text)
|
||||||
|
.await
|
||||||
|
.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
@@ -13,21 +13,15 @@ pub async fn currency_converter(amount: f64, from: &str, to: &str) -> Result<Val
|
|||||||
|
|
||||||
/// Harga emas Antam.
|
/// Harga emas Antam.
|
||||||
pub async fn harga_emas() -> Result<Value, ScrapingError> {
|
pub async fn harga_emas() -> Result<Value, ScrapingError> {
|
||||||
Repo::fetch_harga_emas()
|
Repo::fetch_harga_emas().await.map_err(ScrapingError::Http)
|
||||||
.await
|
|
||||||
.map_err(ScrapingError::Http)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Kurs BCA (jual/beli).
|
/// Kurs BCA (jual/beli).
|
||||||
pub async fn kurs_bca() -> Result<Value, ScrapingError> {
|
pub async fn kurs_bca() -> Result<Value, ScrapingError> {
|
||||||
Repo::fetch_kurs_bca()
|
Repo::fetch_kurs_bca().await.map_err(ScrapingError::Http)
|
||||||
.await
|
|
||||||
.map_err(ScrapingError::Http)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Server info (OS/CPU/RAM/disk).
|
/// Server info (OS/CPU/RAM/disk).
|
||||||
pub async fn server_info() -> Result<Value, ScrapingError> {
|
pub async fn server_info() -> Result<Value, ScrapingError> {
|
||||||
Repo::fetch_server_info()
|
Repo::fetch_server_info().await.map_err(ScrapingError::Http)
|
||||||
.await
|
|
||||||
.map_err(ScrapingError::Http)
|
|
||||||
}
|
}
|
||||||
@@ -1,8 +1,10 @@
|
|||||||
pub mod anime;
|
pub mod anime;
|
||||||
pub mod anime2;
|
pub mod anime2;
|
||||||
pub mod downloader;
|
pub mod downloader;
|
||||||
|
pub mod image;
|
||||||
pub mod komik;
|
pub mod komik;
|
||||||
pub mod misc;
|
pub mod misc;
|
||||||
pub mod proxy;
|
pub mod proxy;
|
||||||
|
pub mod search;
|
||||||
pub mod stalk;
|
pub mod stalk;
|
||||||
pub mod weebs;
|
pub mod tools;
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
//! Application use-cases for search utilities.
|
||||||
|
|
||||||
|
use crate::domain::error::ScrapingError;
|
||||||
|
use crate::infrastructure::repository::search as Repo;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
pub async fn bmkg() -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_bmkg().await.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn jadwal_sholat(kota: &str) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_jadwal_sholat(kota)
|
||||||
|
.await
|
||||||
|
.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn weather(city: &str) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_weather(city).await.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn google(query: &str) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_google(query).await.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn yt_search(query: &str) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_yt_search(query)
|
||||||
|
.await
|
||||||
|
.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
@@ -17,3 +17,10 @@ pub async fn youtube(username: &str) -> Result<Value, ScrapingError> {
|
|||||||
.await
|
.await
|
||||||
.map_err(ScrapingError::Http)
|
.map_err(ScrapingError::Http)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Twitter/X user stalk.
|
||||||
|
pub async fn twitter(username: &str) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_twitter_stalk(username)
|
||||||
|
.await
|
||||||
|
.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
//! Application use-cases for tools.
|
||||||
|
|
||||||
|
use crate::domain::error::ScrapingError;
|
||||||
|
use crate::infrastructure::repository::tools as Repo;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
pub async fn whois(domain: &str) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_whois(domain).await.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn ip_location(ip: &str) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_ip_location(ip)
|
||||||
|
.await
|
||||||
|
.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn tinyurl(url: &str) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_tinyurl(url).await.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn check_hosting(domain: &str) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_check_hosting(domain)
|
||||||
|
.await
|
||||||
|
.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn hargapangan() -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_hargapangan().await.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn cek_resi(resi: String, ekspedisi: Option<String>) -> Result<Value, ScrapingError> {
|
||||||
|
Repo::fetch_cek_resi(&resi, ekspedisi.as_deref())
|
||||||
|
.await
|
||||||
|
.map_err(ScrapingError::Http)
|
||||||
|
}
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
//! Application use-cases for weebs (anime/manga info, waifu, whatanime).
|
|
||||||
|
|
||||||
use crate::domain::error::ScrapingError;
|
|
||||||
use crate::infrastructure::repository::weebs as Repo;
|
|
||||||
use serde_json::Value;
|
|
||||||
|
|
||||||
pub async fn anime_info(query: &str) -> Result<Value, ScrapingError> {
|
|
||||||
Repo::fetch_anime_info(query)
|
|
||||||
.await
|
|
||||||
.map_err(ScrapingError::Http)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn manga_info(query: &str) -> Result<Value, ScrapingError> {
|
|
||||||
Repo::fetch_manga_info(query)
|
|
||||||
.await
|
|
||||||
.map_err(ScrapingError::Http)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn whatanime(url: &str) -> Result<Value, ScrapingError> {
|
|
||||||
Repo::fetch_whatanime(url)
|
|
||||||
.await
|
|
||||||
.map_err(ScrapingError::Http)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn sfw_waifu(tag: &str) -> Result<Value, ScrapingError> {
|
|
||||||
Repo::fetch_sfw_waifu(tag)
|
|
||||||
.await
|
|
||||||
.map_err(ScrapingError::Http)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn nsfw_waifu(tag: &str) -> Result<Value, ScrapingError> {
|
|
||||||
Repo::fetch_nsfw_waifu(tag)
|
|
||||||
.await
|
|
||||||
.map_err(ScrapingError::Http)
|
|
||||||
}
|
|
||||||
@@ -5,6 +5,8 @@
|
|||||||
|
|
||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use aes::cipher::{BlockDecrypt, KeyInit};
|
use aes::cipher::{BlockDecrypt, KeyInit};
|
||||||
@@ -511,6 +513,14 @@ impl DownloaderRepository {
|
|||||||
fetch_youtube_mp4(url, quality).await
|
fetch_youtube_mp4(url, quality).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// YouTube video + audio merged server-side into a single MP4.
|
||||||
|
pub async fn fetch_youtube_merge(
|
||||||
|
url: &str,
|
||||||
|
quality: &str,
|
||||||
|
) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
fetch_youtube_merge_free(url, quality).await
|
||||||
|
}
|
||||||
|
|
||||||
/// YouTube to MP3 via ydlp.yard.id.
|
/// YouTube to MP3 via ydlp.yard.id.
|
||||||
pub async fn download_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
|
pub async fn download_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
fetch_youtube_mp3(url).await
|
fetch_youtube_mp3(url).await
|
||||||
@@ -844,12 +854,26 @@ async fn detect_media_type(url: &str) -> MediaType {
|
|||||||
/// SnapSave parser — extracts Instagram/Facebook media via snapsave.app.
|
/// SnapSave parser — extracts Instagram/Facebook media via snapsave.app.
|
||||||
/// Uses downr.org as fallback for robustness, then Playwright browser scraping.
|
/// Uses downr.org as fallback for robustness, then Playwright browser scraping.
|
||||||
pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, ScrapingError> {
|
pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
// Validate Instagram/Facebook URL
|
// Validate Instagram/Facebook URL.
|
||||||
let valid_fb = regex::Regex::new(r"https?://(web\.|www\.|m\.)(facebook|fb)\.(com|watch)\S+")
|
//
|
||||||
|
// Facebook link shapes (all must pass):
|
||||||
|
// https://www.facebook.com/watch/?v=123
|
||||||
|
// https://fb.watch/abc123/
|
||||||
|
// https://m.facebook.com/video.php?v=123
|
||||||
|
// https://www.facebook.com/100012345678901/videos/1234567890
|
||||||
|
// https://www.facebook.com/reel/123
|
||||||
|
// https://www.facebook.com/story.php?story_fbid=123
|
||||||
|
// https://web.facebook.com/...
|
||||||
|
// NOTE: the old regex required a `www.|m.|web.` subdomain AND ended the
|
||||||
|
// host capture with `(facebook|fb)\.(com|watch)` — that rejected
|
||||||
|
// `facebook.com/watch...` (no subdomain) and `fb.watch/...` entirely.
|
||||||
|
let fb_host = r"(?:[a-z0-9-]*\.)?(?:facebook|fb)\.(?:com|watch)";
|
||||||
|
let valid_fb = regex::Regex::new(&format!(r"https?://{}(?:/|$)", fb_host))
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.is_match(url);
|
.is_match(url);
|
||||||
let valid_ig =
|
let valid_ig = url.contains("instagram.com")
|
||||||
regex::Regex::new(r"https?://(www\.)?instagram\.com/(p|reel|reels|tv|stories)/\S+")
|
|| url.contains("threads.net")
|
||||||
|
|| regex::Regex::new(r"https?://(www\.)?instagram\.com/[^\s]+")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.is_match(url);
|
.is_match(url);
|
||||||
|
|
||||||
@@ -859,7 +883,24 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try downr.org first
|
// Try well-known scrapers in order — Playwright is a LAST resort (heavy,
|
||||||
|
// slow, and its naive response-harvesting returns many broken/404 links
|
||||||
|
// mixed with the one good URL, which is exactly the "link exists but can't
|
||||||
|
// download" bug we've seen).
|
||||||
|
//
|
||||||
|
// 1) yt-dlp facebook extractor — authoritative, returns direct fbcdn
|
||||||
|
// progressive URLs with REAL title/format metadata and only the working
|
||||||
|
// scales (sd + hd, no junk). Also handles /reel, /watch, /videos,
|
||||||
|
// story.php, video.php, fb.watch URL shapes.
|
||||||
|
if let Ok(yt) = run_ytdlp_json(url, &["-f", "bestvideo+bestaudio/best"]).await {
|
||||||
|
if let Some(res) = ytdlp_to_download_result(&yt) {
|
||||||
|
if !res.media.is_empty() {
|
||||||
|
return Ok(res);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) downr.org all-in-one (same provider as Shirokami's /fbdl)
|
||||||
match fetch_all_in_one(url).await {
|
match fetch_all_in_one(url).await {
|
||||||
Ok(result) if !result.media.is_empty() => return Ok(result),
|
Ok(result) if !result.media.is_empty() => return Ok(result),
|
||||||
Ok(_) => {}
|
Ok(_) => {}
|
||||||
@@ -871,7 +912,7 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: use Playwright browser scraping to extract video URLs
|
// 3) Playwright browser scraping to extract video URLs
|
||||||
let platform = if valid_ig { "instagram" } else { "facebook" };
|
let platform = if valid_ig { "instagram" } else { "facebook" };
|
||||||
let data = run_playwright_scraper(url, platform).await?;
|
let data = run_playwright_scraper(url, platform).await?;
|
||||||
|
|
||||||
@@ -889,17 +930,25 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
|||||||
let media_arr = data.get("media").and_then(|v| v.as_array());
|
let media_arr = data.get("media").and_then(|v| v.as_array());
|
||||||
if let Some(medias) = media_arr {
|
if let Some(medias) = media_arr {
|
||||||
for m in medias {
|
for m in medias {
|
||||||
let item = MediaItem {
|
let u = m
|
||||||
url: m
|
|
||||||
.get("url")
|
.get("url")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.unwrap_or("")
|
.unwrap_or("")
|
||||||
.to_string(),
|
.to_string();
|
||||||
|
// Drop Playwright-harvested links that are actually broken
|
||||||
|
// byte-range chunks (fbcdn URLs with bytestart=…byteend=… that
|
||||||
|
// return tiny 200 responses — they 404 or return an HTML error
|
||||||
|
// page when fetched as a whole), and the empty-string url.
|
||||||
|
if u.is_empty() || u.contains("bytestart=") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let item = MediaItem {
|
||||||
|
url: u,
|
||||||
quality: None,
|
quality: None,
|
||||||
file_type: m.get("ext").and_then(|v| v.as_str()).and_then(|e| match e {
|
file_type: m.get("ext").and_then(|v| v.as_str()).map(|e| match e {
|
||||||
"mp4" | "m3u8" => Some(MediaType::Video),
|
"mp4" | "m3u8" => MediaType::Video,
|
||||||
"mp3" | "m4a" => Some(MediaType::Audio),
|
"mp3" | "m4a" => MediaType::Audio,
|
||||||
_ => Some(MediaType::Video),
|
_ => MediaType::Video,
|
||||||
}),
|
}),
|
||||||
extension: m.get("ext").and_then(|v| v.as_str()).map(|s| s.to_string()),
|
extension: m.get("ext").and_then(|v| v.as_str()).map(|s| s.to_string()),
|
||||||
thumbnail: None,
|
thumbnail: None,
|
||||||
@@ -913,6 +962,13 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// De-duplicate by URL (Playwright harvests the same CDN file at many
|
||||||
|
// byte-ranges, and the primary URL appears multiple times).
|
||||||
|
{
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
result.media.retain(|m| seen.insert(m.url.clone()));
|
||||||
|
}
|
||||||
|
|
||||||
if result.media.is_empty() {
|
if result.media.is_empty() {
|
||||||
return Ok(DownloadResult::error(format!(
|
return Ok(DownloadResult::error(format!(
|
||||||
"Failed to extract media from {} — server IP may be blocked by anti-bot protection",
|
"Failed to extract media from {} — server IP may be blocked by anti-bot protection",
|
||||||
@@ -922,6 +978,122 @@ pub(crate) async fn fetch_snapsave(url: &str) -> Result<DownloadResult, Scraping
|
|||||||
|
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Convert a yt-dlp --dump-single-json result into a DownloadResult with the
|
||||||
|
/// direct progressive/adaptive media URLs (only entries that carry a URL).
|
||||||
|
/// Used by the Facebook downloader as the primary provider.
|
||||||
|
fn ytdlp_to_download_result(data: &serde_json::Value) -> Option<DownloadResult> {
|
||||||
|
let title = data
|
||||||
|
.get("title")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let mut result = DownloadResult::success(title);
|
||||||
|
result.provider = Some("yt-dlp".to_string());
|
||||||
|
result.thumbnail = data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
result.duration = data
|
||||||
|
.get("duration")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|d| format!("{}s", d));
|
||||||
|
|
||||||
|
let formats = data.get("formats").and_then(|v| v.as_array());
|
||||||
|
let requested_formats = data.get("requested_formats").and_then(|v| v.as_array());
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
|
||||||
|
// `-f bestvideo+bestaudio/best` puts the merged URLs in requested_formats.
|
||||||
|
// `best` single format lands in top-level url + formats[0].
|
||||||
|
for list in [requested_formats, formats].into_iter().flatten() {
|
||||||
|
for f in list {
|
||||||
|
let Some(furl) = f.get("url").and_then(|v| v.as_str()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !furl.starts_with("http") || !seen.insert(furl.to_string()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let proto = f.get("protocol").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
if proto.contains("m3u8")
|
||||||
|
|| f.get("vcodec").and_then(|v| v.as_str()) == Some("none")
|
||||||
|
&& f.get("acodec").and_then(|v| v.as_str()) != Some("none")
|
||||||
|
{
|
||||||
|
// Skip HLS + audio-only; Facebook's progressive mp4s are direct.
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let ext: String = f
|
||||||
|
.get("ext")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("mp4")
|
||||||
|
.to_string();
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: furl.to_string(),
|
||||||
|
quality: f
|
||||||
|
.get("height")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|h| format!("{}p", h))
|
||||||
|
.or_else(|| {
|
||||||
|
f.get("format_note")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
}),
|
||||||
|
file_type: Some(if ext == "mp4" || ext == "webm" || ext == "mkv" {
|
||||||
|
MediaType::Video
|
||||||
|
} else {
|
||||||
|
MediaType::File
|
||||||
|
}),
|
||||||
|
extension: Some(ext),
|
||||||
|
thumbnail: result.thumbnail.clone(),
|
||||||
|
file_size: f
|
||||||
|
.get("filesize")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(format_filesize),
|
||||||
|
size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
|
||||||
|
frame_width: f
|
||||||
|
.get("width")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|w| w.to_string()),
|
||||||
|
frame_height: f
|
||||||
|
.get("height")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|h| h.to_string()),
|
||||||
|
note: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to the merged top-level URL (bestvideo+bestaudio sets it).
|
||||||
|
if result.media.is_empty() {
|
||||||
|
if let Some(dl_url) = data.get("url").and_then(|v| v.as_str()) {
|
||||||
|
if dl_url.starts_with("http") {
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: dl_url.to_string(),
|
||||||
|
quality: None,
|
||||||
|
file_type: Some(MediaType::Video),
|
||||||
|
extension: data
|
||||||
|
.get("ext")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.or_else(|| Some("mp4".to_string())),
|
||||||
|
thumbnail: None,
|
||||||
|
file_size: data
|
||||||
|
.get("filesize")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(format_filesize),
|
||||||
|
size_bytes: data.get("filesize").and_then(|v| v.as_u64()),
|
||||||
|
frame_width: None,
|
||||||
|
frame_height: None,
|
||||||
|
note: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.media.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(result)
|
||||||
|
}
|
||||||
|
}
|
||||||
/// TikTok via embed-page scraping (primary method).
|
/// TikTok via embed-page scraping (primary method).
|
||||||
/// Scrapes `https://www.tiktok.com/embed/v2/{video_id}` HTML and extracts the
|
/// Scrapes `https://www.tiktok.com/embed/v2/{video_id}` HTML and extracts the
|
||||||
/// direct `v16m.tiktokcdn.com` MP4 URL from the `<video data-testid="play-video">` tag.
|
/// direct `v16m.tiktokcdn.com` MP4 URL from the `<video data-testid="play-video">` tag.
|
||||||
@@ -1402,7 +1574,11 @@ pub async fn fetch_youtube_mp3(url: &str) -> Result<DownloadResult, ScrapingErro
|
|||||||
data.get("abr").and_then(|v| v.as_u64()).unwrap_or(128)
|
data.get("abr").and_then(|v| v.as_u64()).unwrap_or(128)
|
||||||
)),
|
)),
|
||||||
file_type: Some(MediaType::Audio),
|
file_type: Some(MediaType::Audio),
|
||||||
extension: Some("mp3".to_string()),
|
extension: data
|
||||||
|
.get("ext")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.or_else(|| Some("webm".to_string())),
|
||||||
thumbnail: data
|
thumbnail: data
|
||||||
.get("thumbnail")
|
.get("thumbnail")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
@@ -1425,11 +1601,116 @@ pub async fn fetch_youtube_mp4(url: &str, quality: &str) -> Result<DownloadResul
|
|||||||
extract_youtube_id(url)
|
extract_youtube_id(url)
|
||||||
.ok_or_else(|| ScrapingError::Http("Invalid YouTube URL".to_string()))?;
|
.ok_or_else(|| ScrapingError::Http("Invalid YouTube URL".to_string()))?;
|
||||||
|
|
||||||
let fmt = format!(
|
let q = quality.trim_end_matches('p');
|
||||||
"bestvideo[height<={}]/best+bestaudio/best",
|
|
||||||
quality.trim_end_matches('p')
|
// ---------------------------------------------------------------
|
||||||
|
// 1) PREFER a PROGRESSIVE format (video + audio in ONE file).
|
||||||
|
// The `android` player client exposes progressive itags (18/22) that
|
||||||
|
// are NOT rate-limited from datacenter IPs (verified 2026-09-05:
|
||||||
|
// itag 18 downloads at full speed, while audio-only streams crawl).
|
||||||
|
// Fall back to adaptive (bestvideo+bestaudio) below when no
|
||||||
|
// progressive format exists at the requested height.
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
let prog_fmt = format!("best[height<={}][ext=mp4]/best[height<={}]", q, q);
|
||||||
|
if let Ok(prog_data) = run_ytdlp_json(
|
||||||
|
url,
|
||||||
|
&[
|
||||||
|
"--extractor-args",
|
||||||
|
"youtube:player_client=android",
|
||||||
|
"-f",
|
||||||
|
&prog_fmt,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
let prog_url = prog_data
|
||||||
|
.get("url")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let prog_vcodec = prog_data
|
||||||
|
.get("vcodec")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("none");
|
||||||
|
let prog_acodec = prog_data
|
||||||
|
.get("acodec")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("none");
|
||||||
|
let has_audio = prog_acodec != "none" && !prog_acodec.is_empty();
|
||||||
|
let has_video = prog_vcodec != "none" && !prog_vcodec.is_empty();
|
||||||
|
if let Some(purl) = prog_url {
|
||||||
|
if has_audio && has_video && purl.starts_with("http") {
|
||||||
|
let height = prog_data
|
||||||
|
.get("height")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.unwrap_or(360);
|
||||||
|
// Only use the progressive format if it actually meets the
|
||||||
|
// requested quality. `best[height<=Q][ext=mp4]` can return a
|
||||||
|
// lower-height progressive (e.g. 360p when 720p was requested)
|
||||||
|
// because no 720p progressive exists; in that case fall through
|
||||||
|
// to the adaptive (bestvideo+bestaudio) path so the caller
|
||||||
|
// still gets the requested resolution.
|
||||||
|
let q_num: u64 = q.parse().unwrap_or(0);
|
||||||
|
if q_num != 0 && height < q_num {
|
||||||
|
// fall through to adaptive below
|
||||||
|
} else {
|
||||||
|
let ext = prog_data
|
||||||
|
.get("ext")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("mp4");
|
||||||
|
let mut result = DownloadResult::success(
|
||||||
|
prog_data
|
||||||
|
.get("title")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
);
|
);
|
||||||
let data = run_ytdlp_json(url, &["-f", &fmt]).await?;
|
result.author = prog_data
|
||||||
|
.get("uploader")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
result.thumbnail = prog_data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
result.duration = prog_data
|
||||||
|
.get("duration")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|d| format!("{}s", d));
|
||||||
|
result.provider = Some("yt-dlp".to_string());
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: purl,
|
||||||
|
quality: Some(format!("{}p", height)),
|
||||||
|
file_type: Some(MediaType::Video),
|
||||||
|
extension: Some(ext.to_string()),
|
||||||
|
thumbnail: prog_data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
|
file_size: prog_data
|
||||||
|
.get("filesize")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(format_filesize),
|
||||||
|
size_bytes: prog_data.get("filesize").and_then(|v| v.as_u64()),
|
||||||
|
frame_width: prog_data
|
||||||
|
.get("width")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|w| w.to_string()),
|
||||||
|
frame_height: prog_data
|
||||||
|
.get("height")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|h| h.to_string()),
|
||||||
|
note: Some("Progressive MP4 — video and audio in one file".to_string()),
|
||||||
|
});
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let fmt = format!(
|
||||||
|
"bestvideo[height<={}]+bestaudio/best[height<={}]/best",
|
||||||
|
q, q
|
||||||
|
);
|
||||||
|
let data = run_ytdlp_json(url, &["-f", &fmt, "--merge-output-format", "mp4"]).await?;
|
||||||
|
|
||||||
let title = data
|
let title = data
|
||||||
.get("title")
|
.get("title")
|
||||||
@@ -1455,23 +1736,101 @@ pub async fn fetch_youtube_mp4(url: &str, quality: &str) -> Result<DownloadResul
|
|||||||
result.provider = Some("yt-dlp".to_string());
|
result.provider = Some("yt-dlp".to_string());
|
||||||
|
|
||||||
let formats = data.get("formats").and_then(|v| v.as_array());
|
let formats = data.get("formats").and_then(|v| v.as_array());
|
||||||
|
let requested_formats = data.get("requested_formats").and_then(|v| v.as_array());
|
||||||
let mut seen = std::collections::HashSet::new();
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
|
||||||
if let Some(fmts) = formats {
|
// When yt-dlp uses a merged format (bestvideo+bestaudio), the actual
|
||||||
|
// download URLs live in `requested_formats` (video + audio separate URLs).
|
||||||
|
// The top-level `url` is empty and `formats[]` is the FULL list of all
|
||||||
|
// individual format IDs, not just the selected ones.
|
||||||
|
if let Some(rf) = requested_formats {
|
||||||
|
for f in rf {
|
||||||
|
if let (Some(furl), Some(ext_val)) = (
|
||||||
|
f.get("url").and_then(|v| v.as_str()),
|
||||||
|
f.get("ext").and_then(|v| v.as_str()),
|
||||||
|
) {
|
||||||
|
let proto = f.get("protocol").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
// Skip mhtml/storyboard and m3u8/hls streams
|
||||||
|
if ext_val == "mhtml" || proto.contains("m3u8") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let url_string = furl.to_string();
|
||||||
|
if !url_string.starts_with("http") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if seen.insert(url_string.clone()) {
|
||||||
|
let is_video = f
|
||||||
|
.get("vcodec")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|c| c != "none")
|
||||||
|
.unwrap_or(false);
|
||||||
|
let is_audio_only = f
|
||||||
|
.get("acodec")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|c| c != "none")
|
||||||
|
.unwrap_or(false);
|
||||||
|
let fmt_type = if is_video && is_audio_only {
|
||||||
|
MediaType::Video
|
||||||
|
} else if is_audio_only {
|
||||||
|
MediaType::Audio
|
||||||
|
} else if is_video {
|
||||||
|
MediaType::Video
|
||||||
|
} else {
|
||||||
|
MediaType::File
|
||||||
|
};
|
||||||
|
let q = format!("{}p", f.get("height").and_then(|v| v.as_u64()).unwrap_or(0));
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: url_string,
|
||||||
|
quality: Some(q),
|
||||||
|
file_type: Some(fmt_type),
|
||||||
|
extension: Some(ext_val.to_string()),
|
||||||
|
thumbnail: data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
|
file_size: f
|
||||||
|
.get("filesize")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(format_filesize),
|
||||||
|
size_bytes: f.get("filesize").and_then(|v| v.as_u64()),
|
||||||
|
frame_width: f
|
||||||
|
.get("width")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|w| w.to_string()),
|
||||||
|
frame_height: f
|
||||||
|
.get("height")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|h| h.to_string()),
|
||||||
|
note: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if let Some(fmts) = formats {
|
||||||
for f in fmts {
|
for f in fmts {
|
||||||
if let (Some(furl), Some(ext_val)) = (
|
if let (Some(furl), Some(ext_val)) = (
|
||||||
f.get("url").and_then(|v| v.as_str()),
|
f.get("url").and_then(|v| v.as_str()),
|
||||||
f.get("ext").and_then(|v| v.as_str()),
|
f.get("ext").and_then(|v| v.as_str()),
|
||||||
) {
|
) {
|
||||||
if ext_val == "mhtml" {
|
// Skip storyboard, mhtml, m3u8/hls, and audio-only formats.
|
||||||
|
// We only want formats that have both video and audio.
|
||||||
|
let proto = f.get("protocol").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
if ext_val == "mhtml" || proto.contains("m3u8") {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
// Skip audio-only formats (vcodec == "none", acodec != "none")
|
||||||
|
let vcodec = f.get("vcodec").and_then(|v| v.as_str()).unwrap_or("none");
|
||||||
|
let acodec = f.get("acodec").and_then(|v| v.as_str()).unwrap_or("none");
|
||||||
|
if vcodec == "none" && acodec != "none" {
|
||||||
|
continue; // audio-only
|
||||||
|
}
|
||||||
let url_string = furl.to_string();
|
let url_string = furl.to_string();
|
||||||
|
if !url_string.starts_with("http") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if seen.insert(url_string.clone()) {
|
if seen.insert(url_string.clone()) {
|
||||||
let fmt_type = if ext_val == "mp4" || ext_val == "webm" || ext_val == "mkv" {
|
let fmt_type = if ext_val == "mp4" || ext_val == "webm" || ext_val == "mkv" {
|
||||||
MediaType::Video
|
MediaType::Video
|
||||||
} else if ext_val == "mp3" || ext_val == "m4a" || ext_val == "webm" {
|
|
||||||
MediaType::Audio
|
|
||||||
} else {
|
} else {
|
||||||
MediaType::File
|
MediaType::File
|
||||||
};
|
};
|
||||||
@@ -1536,6 +1895,252 @@ pub async fn fetch_youtube_mp4(url: &str, quality: &str) -> Result<DownloadResul
|
|||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// YouTube video + audio SERVER-SIDE MERGE (produces a single MP4 with audio).
|
||||||
|
//
|
||||||
|
// Modern YouTube videos (AV1/VP9, 720p+) only expose separate video-only and
|
||||||
|
// audio-only streams. `fetch_youtube_mp4` returns those two URLs — the caller
|
||||||
|
// would have to merge them manually. This endpoint downloads both streams and
|
||||||
|
// remuxes them into ONE MP4 with `ffmpeg`, stored under the uploads dir and
|
||||||
|
// served via GET /file/yt_merge/{name}. Files older than 2h are cleaned up on
|
||||||
|
// each new merge request to bound disk usage.
|
||||||
|
// ============================================================================
|
||||||
|
pub async fn fetch_youtube_merge_free(
|
||||||
|
url: &str,
|
||||||
|
quality: &str,
|
||||||
|
) -> Result<DownloadResult, ScrapingError> {
|
||||||
|
let video_id = extract_youtube_id(url)
|
||||||
|
.ok_or_else(|| ScrapingError::Http("Invalid YouTube URL".to_string()))?;
|
||||||
|
|
||||||
|
let q = quality.trim_end_matches('p');
|
||||||
|
let fmt = format!(
|
||||||
|
"bestvideo[height<={}]+bestaudio/best[height<={}]/best",
|
||||||
|
q, q
|
||||||
|
);
|
||||||
|
let data = run_ytdlp_json(url, &["-f", &fmt, "--merge-output-format", "mp4"]).await?;
|
||||||
|
|
||||||
|
let title = data
|
||||||
|
.get("title")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let author = data
|
||||||
|
.get("uploader")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let thumbnail = data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
let duration = data
|
||||||
|
.get("duration")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.map(|d| format!("{}s", d));
|
||||||
|
|
||||||
|
// Locate the video-only and audio-only URLs from requested_formats.
|
||||||
|
let requested_formats = data
|
||||||
|
.get("requested_formats")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.ok_or_else(|| ScrapingError::Http("No requested_formats from yt-dlp".to_string()))?;
|
||||||
|
|
||||||
|
let mut video_url: Option<String> = None;
|
||||||
|
let mut audio_url: Option<String> = None;
|
||||||
|
let mut video_ext = "mp4";
|
||||||
|
let mut audio_ext = "webm";
|
||||||
|
for f in requested_formats {
|
||||||
|
let vcodec = f.get("vcodec").and_then(|v| v.as_str()).unwrap_or("none");
|
||||||
|
let acodec = f.get("acodec").and_then(|v| v.as_str()).unwrap_or("none");
|
||||||
|
let f_url = f.get("url").and_then(|v| v.as_str());
|
||||||
|
let proto = f.get("protocol").and_then(|v| v.as_str()).unwrap_or("");
|
||||||
|
if proto.contains("m3u8") {
|
||||||
|
continue; // HLS streams are not directly downloadable via reqwest
|
||||||
|
}
|
||||||
|
if vcodec != "none" && acodec == "none" {
|
||||||
|
if let Some(u) = f_url {
|
||||||
|
video_url = Some(u.to_string());
|
||||||
|
video_ext = f.get("ext").and_then(|v| v.as_str()).unwrap_or("mp4");
|
||||||
|
}
|
||||||
|
} else if vcodec == "none" && acodec != "none" {
|
||||||
|
if let Some(u) = f_url {
|
||||||
|
audio_url = Some(u.to_string());
|
||||||
|
audio_ext = f.get("ext").and_then(|v| v.as_str()).unwrap_or("webm");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let video_url = video_url
|
||||||
|
.ok_or_else(|| ScrapingError::Http("No video stream URL from yt-dlp".to_string()))?;
|
||||||
|
let audio_url = audio_url
|
||||||
|
.ok_or_else(|| ScrapingError::Http("No audio stream URL from yt-dlp".to_string()))?;
|
||||||
|
|
||||||
|
// --- download both streams to the merge temp dir ---
|
||||||
|
let merge_dir = PathBuf::from("/var/lib/scraper/uploads/yt_merge");
|
||||||
|
tokio::fs::create_dir_all(&merge_dir)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ScrapingError::Http(format!("create merge dir failed: {e}")))?;
|
||||||
|
|
||||||
|
// Cleanup stale merged files (> 2h) before writing anything new.
|
||||||
|
cleanup_stale_merges(&merge_dir).await;
|
||||||
|
|
||||||
|
let job_id = uuid::Uuid::new_v4().simple().to_string();
|
||||||
|
let vpath = merge_dir.join(format!("{video_id}_{job_id}_v.{video_ext}"));
|
||||||
|
let apath = merge_dir.join(format!("{video_id}_{job_id}_a.{audio_ext}"));
|
||||||
|
let outpath = merge_dir.join(format!("{video_id}_{job_id}_merged.mp4"));
|
||||||
|
|
||||||
|
download_to_file(&video_url, &vpath).await?;
|
||||||
|
download_to_file(&audio_url, &apath).await?;
|
||||||
|
|
||||||
|
// --- merge with ffmpeg ---
|
||||||
|
let merge_result = tokio::task::spawn_blocking({
|
||||||
|
let vpath = vpath.clone();
|
||||||
|
let apath = apath.clone();
|
||||||
|
let outpath = outpath.clone();
|
||||||
|
move || merge_with_ffmpeg(&vpath, &apath, &outpath)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|e| ScrapingError::Http(format!("ffmpeg join failed: {e}")))?;
|
||||||
|
|
||||||
|
match merge_result {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(e) => {
|
||||||
|
let _ = tokio::fs::remove_file(&vpath).await;
|
||||||
|
let _ = tokio::fs::remove_file(&apath).await;
|
||||||
|
return Err(ScrapingError::Http(format!("ffmpeg merge failed: {e}")));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up the transient inputs now that the merged file exists.
|
||||||
|
let _ = tokio::fs::remove_file(&vpath).await;
|
||||||
|
let _ = tokio::fs::remove_file(&apath).await;
|
||||||
|
|
||||||
|
let file_size = tokio::fs::metadata(&outpath).await.ok().map(|m| m.len());
|
||||||
|
|
||||||
|
let file_name = outpath
|
||||||
|
.file_name()
|
||||||
|
.map(|s| s.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let public_url = format!("/file/yt_merge/{file_name}");
|
||||||
|
|
||||||
|
let mut result = DownloadResult::success(title);
|
||||||
|
result.author = author;
|
||||||
|
result.thumbnail = thumbnail.clone();
|
||||||
|
result.duration = duration;
|
||||||
|
result.provider = Some("yt-dlp+ffmpeg".to_string());
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: public_url,
|
||||||
|
quality: Some(format!("{}p merged", q)),
|
||||||
|
file_type: Some(MediaType::Video),
|
||||||
|
extension: Some("mp4".to_string()),
|
||||||
|
thumbnail: thumbnail.clone(),
|
||||||
|
file_size: file_size.map(format_filesize),
|
||||||
|
size_bytes: file_size,
|
||||||
|
frame_width: None,
|
||||||
|
frame_height: None,
|
||||||
|
note: Some(
|
||||||
|
"Merged server-side: single MP4 file containing both video and audio".to_string(),
|
||||||
|
),
|
||||||
|
});
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Download `url` to `path` with a generous timeout. The yt CDN stream URLs are
|
||||||
|
/// signed and short-lived, so we must download promptly and cannot retry after
|
||||||
|
/// the URL expires.
|
||||||
|
async fn download_to_file(url: &str, path: &Path) -> Result<(), ScrapingError> {
|
||||||
|
// Dedicated client with a longer timeout than the shared 30s one: merged
|
||||||
|
// files are often tens of MB and the yt CDN can be slow from a datacenter.
|
||||||
|
// .no_gzip()/.no_brotli()/.no_deflate() disable reqwest's auto-decompress
|
||||||
|
// so we can write raw bytes — some CDN responses are mis-labelled and fail
|
||||||
|
// the auto-decode step.
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
|
.timeout(Duration::from_secs(300))
|
||||||
|
.connect_timeout(Duration::from_secs(20))
|
||||||
|
.no_gzip()
|
||||||
|
.no_brotli()
|
||||||
|
.no_deflate()
|
||||||
|
.build()
|
||||||
|
.map_err(|e| ScrapingError::Http(format!("stream client build failed: {e}")))?;
|
||||||
|
let resp = client
|
||||||
|
.get(url)
|
||||||
|
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||||
|
.header("Accept-Encoding", "identity")
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| ScrapingError::Http(format!("stream download failed: {e}")))?;
|
||||||
|
|
||||||
|
let status = resp.status();
|
||||||
|
if !status.is_success() {
|
||||||
|
return Err(ScrapingError::Http(format!(
|
||||||
|
"stream download HTTP {status}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let bytes = resp
|
||||||
|
.bytes()
|
||||||
|
.await
|
||||||
|
.map_err(|e| ScrapingError::Http(format!("stream body read failed: {e}")))?;
|
||||||
|
|
||||||
|
tokio::fs::write(path, bytes)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ScrapingError::Http(format!("stream write failed: {e}")))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run ffmpeg to remux the video and audio streams into a single MP4.
|
||||||
|
/// `-c:v copy` avoids a full re-encode (fast); audio is transcoded to AAC so
|
||||||
|
/// the output plays everywhere. `+faststart` moves the moov atom to the front
|
||||||
|
/// so the file streams immediately.
|
||||||
|
fn merge_with_ffmpeg(video: &Path, audio: &Path, out: &Path) -> Result<(), String> {
|
||||||
|
let status = Command::new("/bin/ffmpeg")
|
||||||
|
.args([
|
||||||
|
"-y",
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-i",
|
||||||
|
video.to_str().unwrap_or_default(),
|
||||||
|
"-i",
|
||||||
|
audio.to_str().unwrap_or_default(),
|
||||||
|
"-c:v",
|
||||||
|
"copy",
|
||||||
|
"-c:a",
|
||||||
|
"aac",
|
||||||
|
"-movflags",
|
||||||
|
"+faststart",
|
||||||
|
"-shortest",
|
||||||
|
out.to_str().unwrap_or_default(),
|
||||||
|
])
|
||||||
|
.stdout(Stdio::null())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.status()
|
||||||
|
.map_err(|e| format!("spawn ffmpeg failed: {e}"))?;
|
||||||
|
|
||||||
|
if status.success() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(format!("ffmpeg exited with {status:?}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove merged output + transient files older than 2 hours. Bounded, best
|
||||||
|
/// effort — never fails the request.
|
||||||
|
async fn cleanup_stale_merges(dir: &Path) {
|
||||||
|
let deadline = std::time::SystemTime::now() - std::time::Duration::from_secs(2 * 60 * 60);
|
||||||
|
let Ok(mut entries) = tokio::fs::read_dir(dir).await else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||||
|
let Ok(meta) = entry.metadata().await else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Ok(modified) = meta.modified() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if modified < deadline {
|
||||||
|
let _ = tokio::fs::remove_file(entry.path()).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Spotify downloaders
|
// Spotify downloaders
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
@@ -2804,56 +3409,131 @@ pub async fn fetch_terabox(url: &str) -> Result<DownloadResult, ScrapingError> {
|
|||||||
|
|
||||||
let surl = surl.ok_or_else(|| ScrapingError::Http("SURL not found.".to_string()))?;
|
let surl = surl.ok_or_else(|| ScrapingError::Http("SURL not found.".to_string()))?;
|
||||||
|
|
||||||
|
// Call local TeraBox resolver (Playwright + residential proxy + cookies)
|
||||||
|
let resolver_url = std::env::var("TERABOX_RESOLVER_URL")
|
||||||
|
.unwrap_or_else(|_| "http://127.0.0.1:4092".to_string());
|
||||||
|
let resolve_url = format!("{}/resolve?surl={}", resolver_url, surl);
|
||||||
|
|
||||||
let client = http_client();
|
let client = http_client();
|
||||||
let resp = client
|
let resp = client
|
||||||
.client()
|
.client()
|
||||||
.get("https://tera2.sylyt93.workers.dev/info")
|
.get(&resolve_url)
|
||||||
.query(&[("s", &surl)])
|
|
||||||
.header("origin", "https://www.kauruka.com")
|
|
||||||
.header("referer", "https://www.kauruka.com/")
|
|
||||||
.header(USER_AGENT, "Mozilla/5.0")
|
.header(USER_AGENT, "Mozilla/5.0")
|
||||||
.timeout(Duration::from_secs(15))
|
.timeout(Duration::from_secs(60))
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| ScrapingError::Http(format!("TeraBox info fetch failed: {}", e)))?
|
.map_err(|e| ScrapingError::Http(format!("TeraBox resolver fetch failed: {}", e)))?;
|
||||||
.json::<serde_json::Value>()
|
|
||||||
.await
|
|
||||||
.map_err(|e| ScrapingError::Http(format!("TeraBox JSON parse failed: {}", e)))?;
|
|
||||||
|
|
||||||
if let Some(download_url) = resp.get("url").and_then(|v| v.as_str()) {
|
if !resp.status().is_success() {
|
||||||
let mut result = DownloadResult::success(
|
let status = resp.status();
|
||||||
resp.get("name")
|
let body = resp.text().await.unwrap_or_default();
|
||||||
.and_then(|v| v.as_str())
|
return Ok(DownloadResult::error(format!(
|
||||||
.map(|s| s.to_string()),
|
"TeraBox resolver returned HTTP {}: {}",
|
||||||
);
|
status, body
|
||||||
result.provider = Some("terabox".to_string());
|
)));
|
||||||
result.thumbnail = resp
|
}
|
||||||
.get("img")
|
|
||||||
|
let data: serde_json::Value = resp
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.map_err(|e| ScrapingError::Http(format!("TeraBox resolver JSON parse failed: {}", e)))?;
|
||||||
|
|
||||||
|
if let Some(error) = data.get("error").and_then(|v| v.as_str()) {
|
||||||
|
return Ok(DownloadResult::error(error));
|
||||||
|
}
|
||||||
|
|
||||||
|
let file_name = data
|
||||||
|
.get("file_name")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.map(|s| s.to_string());
|
.map(|s| s.to_string());
|
||||||
|
|
||||||
|
let mut result = DownloadResult::success(file_name.clone());
|
||||||
|
result.provider = Some("terabox".to_string());
|
||||||
|
result.thumbnail = data
|
||||||
|
.get("thumbnail")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
|
||||||
|
// Primary download link.
|
||||||
|
//
|
||||||
|
// IMPORTANT: the resolver's download_link is a raw terabox dlink that
|
||||||
|
// 403s with {"error_code":31045,"user not exists"} when fetched WITHOUT
|
||||||
|
// cookies — even from residential IPs. So we rewrite the public URL to
|
||||||
|
// OUR OWN /proxy/terabox streaming endpoint (which forwards the resolver's
|
||||||
|
// cookie-authenticated stream through the public API). This is the only
|
||||||
|
// way users get a link that actually downloads.
|
||||||
|
// The public API base that users can reach. `CONFIG.urls.site_url` is the
|
||||||
|
// main site (hub → 4003) — NOT the scraper API domain. Use an explicit
|
||||||
|
// env override (TERABOX_PUBLIC_BASE_URL), falling back to the scraper's
|
||||||
|
// own public domain via site_url only when it's clearly the API host.
|
||||||
|
let public_base = std::env::var("TERABOX_PUBLIC_BASE_URL").unwrap_or_else(|_| {
|
||||||
|
let site = crate::config::CONFIG.urls.site_url.clone();
|
||||||
|
if site.contains("scraper") || site.contains("api") {
|
||||||
|
site
|
||||||
|
} else {
|
||||||
|
// default to the scraper API subdomain (this API's public host)
|
||||||
|
"https://api.asepharyana.my.id".to_string()
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let proxy_dl = format!("{}/proxy/terabox?surl={}", public_base, surl);
|
||||||
|
if let Some(download_url) = data.get("download_link").and_then(|v| v.as_str()) {
|
||||||
|
let size_bytes = data.get("file_size").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||||
|
|
||||||
|
// PRIMARY = the cookie-authenticated public proxy. The raw terabox
|
||||||
|
// dlink is only a secondary entry — it 403s without cookies, so it
|
||||||
|
// must NOT be the first link users see.
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: proxy_dl.clone(),
|
||||||
|
quality: None,
|
||||||
|
file_type: Some(MediaType::File),
|
||||||
|
extension: None,
|
||||||
|
thumbnail: None,
|
||||||
|
file_size: Some(format_filesize(size_bytes)),
|
||||||
|
size_bytes: Some(size_bytes),
|
||||||
|
frame_width: None,
|
||||||
|
frame_height: None,
|
||||||
|
note: Some("Direct download via cookie-authenticated proxy (works).".into()),
|
||||||
|
});
|
||||||
result.media.push(MediaItem {
|
result.media.push(MediaItem {
|
||||||
url: download_url.to_string(),
|
url: download_url.to_string(),
|
||||||
quality: None,
|
quality: None,
|
||||||
file_type: Some(MediaType::File),
|
file_type: Some(MediaType::File),
|
||||||
extension: None,
|
extension: None,
|
||||||
thumbnail: None,
|
thumbnail: None,
|
||||||
file_size: None,
|
file_size: Some(format_filesize(size_bytes)),
|
||||||
size_bytes: None,
|
size_bytes: Some(size_bytes),
|
||||||
frame_width: None,
|
frame_width: None,
|
||||||
frame_height: None,
|
frame_height: None,
|
||||||
note: None,
|
note: Some("Raw CDN link (may require a TeraBox session cookie).".into()),
|
||||||
});
|
});
|
||||||
|
|
||||||
Ok(result)
|
// Directory: add child file links
|
||||||
} else {
|
if let Some(files) = data.get("files").and_then(|v| v.as_array()) {
|
||||||
Ok(DownloadResult::error(
|
for f in files {
|
||||||
resp.get("error")
|
if let Some(furl) = f.get("download_link").and_then(|v| v.as_str()) {
|
||||||
|
let fsize = f.get("file_size").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||||
|
result.media.push(MediaItem {
|
||||||
|
url: furl.to_string(),
|
||||||
|
quality: None,
|
||||||
|
file_type: Some(MediaType::File),
|
||||||
|
extension: None,
|
||||||
|
thumbnail: None,
|
||||||
|
file_size: Some(format_filesize(fsize)),
|
||||||
|
size_bytes: Some(fsize),
|
||||||
|
frame_width: None,
|
||||||
|
frame_height: None,
|
||||||
|
note: f
|
||||||
|
.get("file_name")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.unwrap_or("Failed to fetch TeraBox info"),
|
.map(|s| s.to_string()),
|
||||||
))
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn fetch_videy(url: &str) -> Result<DownloadResult, ScrapingError> {
|
pub async fn fetch_videy(url: &str) -> Result<DownloadResult, ScrapingError> {
|
||||||
let id = Url::parse(url)
|
let id = Url::parse(url)
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
//! Infrastructure — Image generation endpoints (brat).
|
||||||
|
//!
|
||||||
|
//! Ported from Shirokami-API `scraper/image/brat.js`.
|
||||||
|
//! brat v2 uses the external brat.siputzx.my.id API (returns image bytes).
|
||||||
|
|
||||||
|
use crate::infrastructure::utils::http_client::http_client;
|
||||||
|
use reqwest::header::USER_AGENT;
|
||||||
|
|
||||||
|
const BRAT_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0 Safari/537.36";
|
||||||
|
|
||||||
|
fn urlencode(s: &str) -> String {
|
||||||
|
s.replace('%', "%25")
|
||||||
|
.replace('&', "%26")
|
||||||
|
.replace('?', "%3F")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch a brat image (static) for the given text.
|
||||||
|
pub async fn fetch_brat(text: &str) -> Result<Vec<u8>, String> {
|
||||||
|
let url = format!("https://brat.siputzx.my.id/image?text={}", urlencode(text));
|
||||||
|
fetch_bytes(&url).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fetch an animated brat GIF for the given text.
|
||||||
|
pub async fn fetch_brat_animated(text: &str) -> Result<Vec<u8>, String> {
|
||||||
|
let url = format!("https://brat.siputzx.my.id/gif?text={}", urlencode(text));
|
||||||
|
fetch_bytes(&url).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_bytes(url: &str) -> Result<Vec<u8>, String> {
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(url)
|
||||||
|
.header(USER_AGENT, BRAT_UA)
|
||||||
|
.header("X-Forwarded-For", "104.28.209.35")
|
||||||
|
.header("X-Real-IP", "104.28.209.35")
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP: {}", e))?;
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(format!("HTTP {}", resp.status()));
|
||||||
|
}
|
||||||
|
resp.bytes()
|
||||||
|
.await
|
||||||
|
.map(|b| b.to_vec())
|
||||||
|
.map_err(|e| format!("Body: {}", e))
|
||||||
|
}
|
||||||
@@ -68,10 +68,7 @@ pub async fn fetch_currency_converter(amount: f64, from: &str, to: &str) -> Resu
|
|||||||
.collect::<String>();
|
.collect::<String>();
|
||||||
|
|
||||||
// Look for patterns like "15,885.00 IDR" in body text
|
// Look for patterns like "15,885.00 IDR" in body text
|
||||||
let re2 = Regex::new(&format!(
|
let re2 = Regex::new(&format!(r"([0-9,.]+)\s+{}", to.to_uppercase()))
|
||||||
r"([0-9,.]+)\s+{}",
|
|
||||||
to.to_uppercase()
|
|
||||||
))
|
|
||||||
.map_err(|e| format!("Regex compile: {}", e))?;
|
.map_err(|e| format!("Regex compile: {}", e))?;
|
||||||
|
|
||||||
if let Some(caps) = re2.captures(&text) {
|
if let Some(caps) = re2.captures(&text) {
|
||||||
@@ -228,7 +225,11 @@ pub async fn fetch_kurs_bca() -> Result<Value, String> {
|
|||||||
.map(|td| td.text().collect::<String>().trim().to_string())
|
.map(|td| td.text().collect::<String>().trim().to_string())
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if cols.len() >= 3 && !cols[0].is_empty() && !cols[1].is_empty() && !cols[2].is_empty() {
|
if cols.len() >= 3
|
||||||
|
&& !cols[0].is_empty()
|
||||||
|
&& !cols[1].is_empty()
|
||||||
|
&& !cols[2].is_empty()
|
||||||
|
{
|
||||||
kurs_data.push(json!({
|
kurs_data.push(json!({
|
||||||
"currency": cols[0],
|
"currency": cols[0],
|
||||||
"jual": cols[1],
|
"jual": cols[1],
|
||||||
|
|||||||
@@ -1,12 +1,14 @@
|
|||||||
pub mod alqanime;
|
pub mod alqanime;
|
||||||
pub mod downloader;
|
pub mod downloader;
|
||||||
|
pub mod image;
|
||||||
pub mod komik;
|
pub mod komik;
|
||||||
pub mod misc;
|
pub mod misc;
|
||||||
pub mod otakudesu;
|
pub mod otakudesu;
|
||||||
pub mod parsers;
|
pub mod parsers;
|
||||||
pub mod proxy;
|
pub mod proxy;
|
||||||
|
pub mod search;
|
||||||
pub mod stalk;
|
pub mod stalk;
|
||||||
pub mod weebs;
|
pub mod tools;
|
||||||
|
|
||||||
pub use alqanime::AlqanimeRepository;
|
pub use alqanime::AlqanimeRepository;
|
||||||
pub use downloader::DownloaderRepository;
|
pub use downloader::DownloaderRepository;
|
||||||
|
|||||||
@@ -0,0 +1,289 @@
|
|||||||
|
//! Infrastructure — Search utilities.
|
||||||
|
//!
|
||||||
|
//! Ported from Shirokami-API `scraper/search/*.js`:
|
||||||
|
//! bmkg (BMKG earthquake), jadwal-sholat (myquran.com), weather (OpenWeather),
|
||||||
|
//! google (HTML scrape), yt (via yt-dlp flat-playlist).
|
||||||
|
|
||||||
|
use crate::infrastructure::utils::http_client::http_client;
|
||||||
|
use reqwest::header::USER_AGENT;
|
||||||
|
use scraper::{Html, Selector};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use std::process::Stdio;
|
||||||
|
|
||||||
|
/// BMKG Indonesia earthquake info (auto-gempa, terkini, dirasakan).
|
||||||
|
pub async fn fetch_bmkg() -> Result<Value, String> {
|
||||||
|
let urls = [
|
||||||
|
"https://data.bmkg.go.id/DataMKG/TEWS/autogempa.json",
|
||||||
|
"https://data.bmkg.go.id/DataMKG/TEWS/gempaterkini.json",
|
||||||
|
"https://data.bmkg.go.id/DataMKG/TEWS/gempadirasakan.json",
|
||||||
|
];
|
||||||
|
let mut auto = Value::Null;
|
||||||
|
let mut terkini: Vec<Value> = Vec::new();
|
||||||
|
let mut dirasakan: Vec<Value> = Vec::new();
|
||||||
|
for url in urls {
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(url)
|
||||||
|
.header(
|
||||||
|
USER_AGENT,
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
|
||||||
|
)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP error: {}", e))?;
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(format!("BMKG returned HTTP {}", resp.status()));
|
||||||
|
}
|
||||||
|
let data: Value = resp
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("JSON parse: {}", e))?;
|
||||||
|
if url.contains("autogempa") {
|
||||||
|
auto = data
|
||||||
|
.pointer("/Infogempa/gempa")
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or(Value::Null);
|
||||||
|
} else if url.contains("gempaterkini") {
|
||||||
|
terkini = data
|
||||||
|
.pointer("/Infogempa/gempa")
|
||||||
|
.and_then(|g| g.as_array())
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_default();
|
||||||
|
} else {
|
||||||
|
dirasakan = data
|
||||||
|
.pointer("/Infogempa/gempa")
|
||||||
|
.and_then(|g| g.as_array())
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_default();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(json!({
|
||||||
|
"autogempa": auto,
|
||||||
|
"gempaterkini": terkini,
|
||||||
|
"gempadirasakan": dirasakan,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Jadwal sholat from myquran.com.
|
||||||
|
pub async fn fetch_jadwal_sholat(kota: &str) -> Result<Value, String> {
|
||||||
|
// 1. Find city id
|
||||||
|
let search_url = format!(
|
||||||
|
"https://api.myquran.com/v2/sholat/kota/cari/{}",
|
||||||
|
urlencode(kota)
|
||||||
|
);
|
||||||
|
let city_data = get_json(&search_url).await?;
|
||||||
|
if city_data
|
||||||
|
.get("status")
|
||||||
|
.and_then(|s| s.as_bool())
|
||||||
|
.unwrap_or(false)
|
||||||
|
!= true
|
||||||
|
{
|
||||||
|
return Ok(json!({ "error": "Kota tidak ditemukan" }));
|
||||||
|
}
|
||||||
|
let id_list = city_data
|
||||||
|
.get("data")
|
||||||
|
.and_then(|d| d.as_array())
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_default();
|
||||||
|
if id_list.is_empty() {
|
||||||
|
return Ok(json!({ "error": "Kota tidak ditemukan" }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Today's date (Asia/Jakarta). Use chrono with a fixed offset approximation.
|
||||||
|
let now = chrono::Local::now();
|
||||||
|
let (year, month, day) = (
|
||||||
|
now.format("%Y").to_string(),
|
||||||
|
now.format("%m").to_string(),
|
||||||
|
now.format("%d").to_string(),
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut results: Vec<Value> = Vec::new();
|
||||||
|
for city in id_list.iter().take(5) {
|
||||||
|
let cid = city.get("id").and_then(|i| i.as_str()).unwrap_or("");
|
||||||
|
let jadwal_url = format!(
|
||||||
|
"https://api.myquran.com/v2/sholat/jadwal/{}/{}/{}/{}",
|
||||||
|
cid, year, month, day
|
||||||
|
);
|
||||||
|
if let Ok(sched) = get_json(&jadwal_url).await {
|
||||||
|
if let Some(d) = sched.get("data") {
|
||||||
|
results.push(json!({
|
||||||
|
"lokasi": d.get("lokasi").cloned().unwrap_or(Value::Null),
|
||||||
|
"daerah": d.get("daerah").cloned().unwrap_or(Value::Null),
|
||||||
|
"jadwal": d.get("jadwal").cloned().unwrap_or(Value::Null),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if results.is_empty() {
|
||||||
|
return Ok(json!({ "error": "Tidak dapat mengambil jadwal" }));
|
||||||
|
}
|
||||||
|
Ok(json!({ "total": results.len(), "schedules": results }))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// OpenWeather current weather (uses env OPENWEATHER_API_KEY, falls back to source's key).
|
||||||
|
pub async fn fetch_weather(city: &str) -> Result<Value, String> {
|
||||||
|
let key = std::env::var("OPENWEATHER_API_KEY")
|
||||||
|
.unwrap_or_else(|_| "060a6bcfa19809c2cd4d97a212b19273".to_string());
|
||||||
|
let url = format!(
|
||||||
|
"https://api.openweathermap.org/data/2.5/weather?q={}&units=metric&appid={}",
|
||||||
|
urlencode(city),
|
||||||
|
key
|
||||||
|
);
|
||||||
|
get_json(&url).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Google web search via HTML scraping.
|
||||||
|
pub async fn fetch_google(query: &str) -> Result<Value, String> {
|
||||||
|
let url = format!(
|
||||||
|
"https://www.google.com/search?q={}&safe=off&hl=en&gl=us",
|
||||||
|
urlencode(query)
|
||||||
|
);
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(&url)
|
||||||
|
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
||||||
|
.header("Accept-Language", "en-US")
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP error: {}", e))?;
|
||||||
|
let body = resp
|
||||||
|
.text()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("Failed to read body: {}", e))?;
|
||||||
|
|
||||||
|
let document = Html::parse_document(&body);
|
||||||
|
let result_sel =
|
||||||
|
Selector::parse("div.g, div[data-hveid]").unwrap_or_else(|_| Selector::parse("a").unwrap());
|
||||||
|
let link_sel = Selector::parse("a").unwrap();
|
||||||
|
let h3_sel = Selector::parse("h3").unwrap();
|
||||||
|
|
||||||
|
let mut results: Vec<Value> = Vec::new();
|
||||||
|
for node in document.select(&result_sel) {
|
||||||
|
let link = node
|
||||||
|
.select(&link_sel)
|
||||||
|
.find_map(|a| a.value().attr("href"))
|
||||||
|
.and_then(|h| h.strip_prefix("/url?q="))
|
||||||
|
.map(|h| h.split('&').next().unwrap_or(h).to_string());
|
||||||
|
let title = node
|
||||||
|
.select(&h3_sel)
|
||||||
|
.next()
|
||||||
|
.map(|h| h.text().collect::<String>());
|
||||||
|
if title.is_some() && link.is_some() {
|
||||||
|
results.push(json!({
|
||||||
|
"title": title.unwrap_or_default(),
|
||||||
|
"description": "",
|
||||||
|
"link": link.unwrap_or_default(),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
if results.len() >= 10 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(json!(results))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// YouTube search via yt-dlp flat playlist (real results, scraping).
|
||||||
|
pub async fn fetch_yt_search(query: &str) -> Result<Value, String> {
|
||||||
|
let q = format!("ytsearch15:{}", query);
|
||||||
|
let output = run_ytdlp(&q).await?;
|
||||||
|
|
||||||
|
let mut videos: Vec<Value> = Vec::new();
|
||||||
|
let arr: Vec<Value> = serde_json::from_str(&output).unwrap_or_default();
|
||||||
|
for v in arr.iter().take(15) {
|
||||||
|
videos.push(json!({
|
||||||
|
"title": v.get("title").cloned().unwrap_or(Value::Null),
|
||||||
|
"id": v.get("id").cloned().unwrap_or(Value::Null),
|
||||||
|
"url": v.get("webpage_url").cloned().unwrap_or(Value::Null),
|
||||||
|
"thumbnail": v.get("thumbnail").cloned().unwrap_or(Value::Null),
|
||||||
|
"duration": v.get("duration").cloned().unwrap_or(Value::Null),
|
||||||
|
"views": v.get("view_count").cloned().unwrap_or(Value::Null),
|
||||||
|
"author": {
|
||||||
|
"name": v.pointer("/channel").cloned().unwrap_or(Value::Null),
|
||||||
|
"url": v.get("channel_url").cloned().unwrap_or(Value::Null),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(json!({ "total": videos.len(), "videos": videos }))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Helpers
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
async fn get_json(url: &str) -> Result<Value, String> {
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(url)
|
||||||
|
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP error: {}", e))?;
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(format!("HTTP {} from upstream", resp.status()));
|
||||||
|
}
|
||||||
|
resp.json::<Value>()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("JSON parse: {}", e))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run yt-dlp with `--dump-json --flat-playlist` for the given search/query arg.
|
||||||
|
async fn run_ytdlp(query_arg: &str) -> Result<String, String> {
|
||||||
|
let bins = [
|
||||||
|
"/usr/local/bin/yt-dlp-native",
|
||||||
|
"/home/code/hermes-agent/.venv/bin/yt-dlp",
|
||||||
|
"yt-dlp",
|
||||||
|
];
|
||||||
|
let bin = bins
|
||||||
|
.iter()
|
||||||
|
.find(|b| std::path::Path::new(b).exists() || **b == "yt-dlp")
|
||||||
|
.copied()
|
||||||
|
.unwrap_or("yt-dlp");
|
||||||
|
|
||||||
|
let child = tokio::process::Command::new(bin)
|
||||||
|
.args(["--dump-json", "--flat-playlist", "--no-warnings", query_arg])
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::null())
|
||||||
|
.spawn()
|
||||||
|
.map_err(|e| format!("Failed to spawn yt-dlp: {}", e))?;
|
||||||
|
|
||||||
|
let output = child
|
||||||
|
.wait_with_output()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("yt-dlp failed: {}", e))?;
|
||||||
|
|
||||||
|
let stdout = String::from_utf8(output.stdout).map_err(|e| format!("Invalid UTF-8: {}", e))?;
|
||||||
|
// `--flat-playlist` outputs multiple JSON lines (one per video).
|
||||||
|
let mut joined = String::new();
|
||||||
|
joined.push('[');
|
||||||
|
let mut first = true;
|
||||||
|
for line in stdout.lines() {
|
||||||
|
let line = line.trim();
|
||||||
|
if line.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !first {
|
||||||
|
joined.push(',');
|
||||||
|
}
|
||||||
|
joined.push_str(line);
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
joined.push(']');
|
||||||
|
Ok(joined)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn urlencode(s: &str) -> String {
|
||||||
|
let mut out = String::new();
|
||||||
|
for b in s.bytes() {
|
||||||
|
match b {
|
||||||
|
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
||||||
|
out.push(b as char)
|
||||||
|
}
|
||||||
|
b' ' => out.push_str("%20"),
|
||||||
|
_ => out.push_str(&format!("%{:02X}", b)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
//! Infrastructure — Stalk utilities (github, youtube, instagram, tiktok, twitter).
|
//! Infrastructure — Stalk utilities (github, youtube, twitter).
|
||||||
//!
|
//!
|
||||||
//! Ported from Shirokami-API `scraper/stalk/*.js`.
|
//! Ported from Shirokami-API `scraper/stalk/*.js`.
|
||||||
|
|
||||||
@@ -81,8 +81,13 @@ pub async fn fetch_youtube_stalk(username: &str) -> Result<Value, String> {
|
|||||||
.map_err(|e| format!("Failed to read body: {}", e))?;
|
.map_err(|e| format!("Failed to read body: {}", e))?;
|
||||||
|
|
||||||
// Extract var ytInitialData = {...} via brace matching (JSON may contain ";").
|
// Extract var ytInitialData = {...} via brace matching (JSON may contain ";").
|
||||||
let start = body.find("var ytInitialData = ").ok_or_else(|| "ytInitialData not found".to_string())?;
|
let start = body
|
||||||
let brace = body[start..].find('{').ok_or_else(|| "ytInitialData parse".to_string())? + start;
|
.find("var ytInitialData = ")
|
||||||
|
.ok_or_else(|| "ytInitialData not found".to_string())?;
|
||||||
|
let brace = body[start..]
|
||||||
|
.find('{')
|
||||||
|
.ok_or_else(|| "ytInitialData parse".to_string())?
|
||||||
|
+ start;
|
||||||
let json_str = extract_balanced_json(&body, brace)?;
|
let json_str = extract_balanced_json(&body, brace)?;
|
||||||
let parsed: Value =
|
let parsed: Value =
|
||||||
serde_json::from_str(&json_str).map_err(|e| format!("JSON parse: {}", e))?;
|
serde_json::from_str(&json_str).map_err(|e| format!("JSON parse: {}", e))?;
|
||||||
@@ -99,7 +104,10 @@ pub async fn fetch_youtube_stalk(username: &str) -> Result<Value, String> {
|
|||||||
"isFamilySafe": Value::Null,
|
"isFamilySafe": Value::Null,
|
||||||
});
|
});
|
||||||
|
|
||||||
if let Some(meta) = parsed.get("metadata").and_then(|m| m.get("channelMetadataRenderer")) {
|
if let Some(meta) = parsed
|
||||||
|
.get("metadata")
|
||||||
|
.and_then(|m| m.get("channelMetadataRenderer"))
|
||||||
|
{
|
||||||
let mut set = |dst: &str, src: &str| {
|
let mut set = |dst: &str, src: &str| {
|
||||||
if let Some(v) = meta.get(src) {
|
if let Some(v) = meta.get(src) {
|
||||||
channel[dst] = v.clone();
|
channel[dst] = v.clone();
|
||||||
@@ -112,7 +120,10 @@ pub async fn fetch_youtube_stalk(username: &str) -> Result<Value, String> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Subscriber/video count from header.pageHeaderRenderer
|
// Subscriber/video count from header.pageHeaderRenderer
|
||||||
if let Some(header) = parsed.get("header").and_then(|h| h.get("pageHeaderRenderer")) {
|
if let Some(header) = parsed
|
||||||
|
.get("header")
|
||||||
|
.and_then(|h| h.get("pageHeaderRenderer"))
|
||||||
|
{
|
||||||
if let Some(rows) = header
|
if let Some(rows) = header
|
||||||
.pointer("/content/pageHeaderViewModel/metadata/contentMetadataViewModel/metadataRows")
|
.pointer("/content/pageHeaderViewModel/metadata/contentMetadataViewModel/metadataRows")
|
||||||
{
|
{
|
||||||
@@ -168,7 +179,7 @@ pub async fn fetch_youtube_stalk(username: &str) -> Result<Value, String> {
|
|||||||
.and_then(|t| t.as_array())
|
.and_then(|t| t.as_array())
|
||||||
{
|
{
|
||||||
if let Some(contents) = tabs
|
if let Some(contents) = tabs
|
||||||
.get(0)
|
.first()
|
||||||
.and_then(|t| t.pointer("/tabRenderer/content/sectionListRenderer/contents"))
|
.and_then(|t| t.pointer("/tabRenderer/content/sectionListRenderer/contents"))
|
||||||
.and_then(|c| c.as_array())
|
.and_then(|c| c.as_array())
|
||||||
{
|
{
|
||||||
@@ -212,8 +223,8 @@ fn extract_balanced_json(body: &str, open_brace: usize) -> Result<String, String
|
|||||||
let mut in_str = false;
|
let mut in_str = false;
|
||||||
let mut escaped = false;
|
let mut escaped = false;
|
||||||
let mut close = None;
|
let mut close = None;
|
||||||
for i in open_brace..bytes.len() {
|
for (i, b) in bytes.iter().enumerate().skip(open_brace) {
|
||||||
let c = bytes[i] as char;
|
let c = *b as char;
|
||||||
if in_str {
|
if in_str {
|
||||||
if escaped {
|
if escaped {
|
||||||
escaped = false;
|
escaped = false;
|
||||||
@@ -293,8 +304,10 @@ fn video_from_video_renderer(vd: &Value) -> Value {
|
|||||||
.pointer("/thumbnailOverlays")
|
.pointer("/thumbnailOverlays")
|
||||||
.and_then(|o| o.as_array())
|
.and_then(|o| o.as_array())
|
||||||
.and_then(|arr| {
|
.and_then(|arr| {
|
||||||
arr.iter()
|
arr.iter().find_map(|ov| {
|
||||||
.find_map(|ov| ov.pointer("/thumbnailOverlayTimeStatusRenderer/text/simpleText").cloned())
|
ov.pointer("/thumbnailOverlayTimeStatusRenderer/text/simpleText")
|
||||||
|
.cloned()
|
||||||
|
})
|
||||||
})
|
})
|
||||||
{
|
{
|
||||||
v["duration"] = dur;
|
v["duration"] = dur;
|
||||||
@@ -328,7 +341,7 @@ fn video_from_lockup(lv: &Value) -> Value {
|
|||||||
.pointer("/metadata/lockupMetadataViewModel/metadata/contentMetadataViewModel/metadataRows/0/metadataParts")
|
.pointer("/metadata/lockupMetadataViewModel/metadata/contentMetadataViewModel/metadataRows/0/metadataParts")
|
||||||
.and_then(|p| p.as_array())
|
.and_then(|p| p.as_array())
|
||||||
{
|
{
|
||||||
if let Some(p0) = parts.get(0) {
|
if let Some(p0) = parts.first() {
|
||||||
view_count = p0
|
view_count = p0
|
||||||
.pointer("/text/content")
|
.pointer("/text/content")
|
||||||
.cloned()
|
.cloned()
|
||||||
@@ -393,11 +406,53 @@ fn video_from_grid_video(gv: &Value) -> Value {
|
|||||||
.pointer("/thumbnailOverlays")
|
.pointer("/thumbnailOverlays")
|
||||||
.and_then(|o| o.as_array())
|
.and_then(|o| o.as_array())
|
||||||
.and_then(|arr| {
|
.and_then(|arr| {
|
||||||
arr.iter()
|
arr.iter().find_map(|ov| {
|
||||||
.find_map(|ov| ov.pointer("/thumbnailOverlayTimeStatusRenderer/text/simpleText").cloned())
|
ov.pointer("/thumbnailOverlayTimeStatusRenderer/text/simpleText")
|
||||||
|
.cloned()
|
||||||
|
})
|
||||||
})
|
})
|
||||||
{
|
{
|
||||||
v["duration"] = dur;
|
v["duration"] = dur;
|
||||||
}
|
}
|
||||||
v
|
v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Twitter — fxtwitter API
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
pub async fn fetch_twitter_stalk(username: &str) -> Result<Value, String> {
|
||||||
|
let url = format!("https://api.fxtwitter.com/{}", username);
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(&url)
|
||||||
|
.header(
|
||||||
|
USER_AGENT,
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/130.0 Safari/537.36",
|
||||||
|
)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP: {}", e))?;
|
||||||
|
let data: Value = resp.json().await.map_err(|e| format!("JSON: {}", e))?;
|
||||||
|
|
||||||
|
// Transform to the source's shape
|
||||||
|
let u = &data["user"];
|
||||||
|
Ok(json!({
|
||||||
|
"message": data.get("message").cloned().unwrap_or(Value::Null),
|
||||||
|
"user": {
|
||||||
|
"id": u.get("id").cloned().unwrap_or(Value::Null),
|
||||||
|
"url": u.get("url").cloned().unwrap_or(Value::Null),
|
||||||
|
"screen_name": u.get("screen_name").cloned().unwrap_or(Value::Null),
|
||||||
|
"name": u.get("name").cloned().unwrap_or(Value::Null),
|
||||||
|
"location": u.get("location").cloned().unwrap_or(Value::Null),
|
||||||
|
"description": u.get("description").cloned().unwrap_or(Value::Null),
|
||||||
|
"followers": u.get("followers").cloned().unwrap_or(Value::Null),
|
||||||
|
"following": u.get("following").cloned().unwrap_or(Value::Null),
|
||||||
|
"likes": u.get("likes").cloned().unwrap_or(Value::Null),
|
||||||
|
"banner_url": u.get("banner_url").cloned().unwrap_or(Value::Null),
|
||||||
|
"avatar_url": u.get("avatar_url").cloned().unwrap_or(Value::Null),
|
||||||
|
"joined_at": u.get("joined").cloned().unwrap_or(Value::Null),
|
||||||
|
"website": u.get("website").cloned().unwrap_or(Value::Null),
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
@@ -0,0 +1,438 @@
|
|||||||
|
//! Infrastructure — Tool utilities.
|
||||||
|
//!
|
||||||
|
//! Ported from Shirokami-API `scraper/tool/*.js` and `scraper/tool/*.js`:
|
||||||
|
//! whois, ip-location, tinyurl, check-hosting, cek-resi, hargapangan.
|
||||||
|
|
||||||
|
use crate::infrastructure::utils::http_client::http_client;
|
||||||
|
use regex::Regex;
|
||||||
|
use reqwest::header::USER_AGENT;
|
||||||
|
use scraper::{Html, Selector};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use urlencoding::encode as urlencode;
|
||||||
|
|
||||||
|
const CHROME_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36";
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// WHOIS — RDAP lookup (whois.com HTML `.df-raw` is dead/JS-rendered since 2026;
|
||||||
|
// RDAP is the structured, reliable server-side source for the same data).
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
pub async fn fetch_whois(domain: &str) -> Result<Value, String> {
|
||||||
|
let domain = domain.to_lowercase();
|
||||||
|
let tld = domain.rsplit('.').next().unwrap_or("com");
|
||||||
|
let base = match tld {
|
||||||
|
"com" | "net" => "https://rdap.verisign.com",
|
||||||
|
"org" => "https://rdap.org",
|
||||||
|
"io" => "https://rdap.identitydigital.services",
|
||||||
|
"co" => "https://rdap.nic.co",
|
||||||
|
"id" => "https://rdap.idnnic.id",
|
||||||
|
_ => "https://rdap.org", // generic bootstrap
|
||||||
|
};
|
||||||
|
let url = if base == "https://rdap.verisign.com" {
|
||||||
|
format!("{}/{}/v1/domain/{}", base, tld, domain)
|
||||||
|
} else {
|
||||||
|
format!("{}/domain/{}", base, domain)
|
||||||
|
};
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(&url)
|
||||||
|
.header(
|
||||||
|
USER_AGENT,
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0 Safari/537.36",
|
||||||
|
)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP: {}", e))?;
|
||||||
|
if resp.status() == reqwest::StatusCode::NOT_FOUND {
|
||||||
|
return Ok(
|
||||||
|
json!({"domain": domain, "available": true, "valid": true, "data": "No data", "rawData": "No data"}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Ok(
|
||||||
|
json!({"domain": domain, "valid": false, "data": "RDAP unavailable", "rawData": "No data"}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let data: Value = resp.json().await.map_err(|e| format!("JSON: {}", e))?;
|
||||||
|
|
||||||
|
let event_date = |action: &str| -> String {
|
||||||
|
data.get("events")
|
||||||
|
.and_then(|e| e.as_array())
|
||||||
|
.map(|events| {
|
||||||
|
events
|
||||||
|
.iter()
|
||||||
|
.find(|ev| ev.get("eventAction").and_then(|a| a.as_str()) == Some(action))
|
||||||
|
.and_then(|ev| ev.get("eventDate").and_then(|d| d.as_str()))
|
||||||
|
.unwrap_or("No data")
|
||||||
|
.to_string()
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| "No data".to_string())
|
||||||
|
};
|
||||||
|
|
||||||
|
let domain_name = data
|
||||||
|
.get("ldhName")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or(&domain)
|
||||||
|
.to_string();
|
||||||
|
let domain_id = data
|
||||||
|
.get("handle")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("No data")
|
||||||
|
.to_string();
|
||||||
|
let creation = event_date("registration");
|
||||||
|
let expiration = event_date("expiration");
|
||||||
|
let updated = event_date("last changed");
|
||||||
|
|
||||||
|
// Registrar name from the registrar entity's vcardArray
|
||||||
|
let registrar = data
|
||||||
|
.get("entities")
|
||||||
|
.and_then(|e| e.as_array())
|
||||||
|
.map(|entities| {
|
||||||
|
entities
|
||||||
|
.iter()
|
||||||
|
.find(|en| {
|
||||||
|
en.get("roles")
|
||||||
|
.and_then(|r| r.as_array())
|
||||||
|
.map(|roles| roles.iter().any(|ro| ro.as_str() == Some("registrar")))
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
|
.and_then(|reg| {
|
||||||
|
reg.get("vcardArray")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.and_then(|arr| arr.get(1))
|
||||||
|
.and_then(|props| props.as_array())
|
||||||
|
.and_then(|props| {
|
||||||
|
props.iter().find_map(|p| {
|
||||||
|
let pa = p.as_array()?;
|
||||||
|
if pa.get(0)?.as_str() == Some("fn") {
|
||||||
|
pa.get(3).and_then(|v| v.as_str()).map(|s| s.to_string())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| "No data".to_string())
|
||||||
|
})
|
||||||
|
.unwrap_or_else(|| "No data".to_string());
|
||||||
|
|
||||||
|
Ok(json!({
|
||||||
|
"domain": domain,
|
||||||
|
"available": false,
|
||||||
|
"valid": true,
|
||||||
|
"data": {
|
||||||
|
"domainName": domain_name,
|
||||||
|
"registryDomainId": domain_id,
|
||||||
|
"creationDate": creation,
|
||||||
|
"expirationDate": expiration,
|
||||||
|
"updatedDate": updated,
|
||||||
|
"country": "No data",
|
||||||
|
"registrar": registrar,
|
||||||
|
},
|
||||||
|
"rawData": serde_json::to_string(&data).unwrap_or_default(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// IP Location — ipapi.co JSON
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
pub async fn fetch_ip_location(ip: &str) -> Result<Value, String> {
|
||||||
|
let url = format!("https://ipapi.co/{}/json/", ip);
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(&url)
|
||||||
|
.header(USER_AGENT, "nodejs-ipapi-v1.02")
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP: {}", e))?;
|
||||||
|
resp.json().await.map_err(|e| format!("JSON: {}", e))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// TinyURL — tinyurl.com API
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
pub async fn fetch_tinyurl(url: &str) -> Result<Value, String> {
|
||||||
|
let api_url = format!("https://tinyurl.com/api-create.php?url={}", urlencode(url));
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(&api_url)
|
||||||
|
.header(USER_AGENT, CHROME_UA)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP: {}", e))?;
|
||||||
|
let short_url = resp.text().await.map_err(|e| format!("Body: {}", e))?;
|
||||||
|
Ok(json!({"shortUrl": short_url.trim()}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Check Hosting — hosting-checker.net API
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
pub async fn fetch_check_hosting(domain: &str) -> Result<Value, String> {
|
||||||
|
let cd = domain.to_lowercase().trim().to_string();
|
||||||
|
// Strip scheme and www
|
||||||
|
let cd = cd
|
||||||
|
.trim_start_matches("http://")
|
||||||
|
.trim_start_matches("https://")
|
||||||
|
.trim_start_matches("www.")
|
||||||
|
.to_string();
|
||||||
|
let domain_re =
|
||||||
|
Regex::new(r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$").unwrap();
|
||||||
|
if !domain_re.is_match(&cd) {
|
||||||
|
return Err("Invalid domain format".into());
|
||||||
|
}
|
||||||
|
let url = format!("https://hosting-checker.net/api/hosting/{}", cd);
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(&url)
|
||||||
|
.header("authority", "hosting-checker.net")
|
||||||
|
.header(USER_AGENT, "Postify/1.0.0")
|
||||||
|
.header(
|
||||||
|
"referer",
|
||||||
|
format!("https://hosting-checker.net/websites/{}", cd),
|
||||||
|
)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP: {}", e))?;
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(format!("HTTP {}", resp.status()));
|
||||||
|
}
|
||||||
|
let data: Value = resp.json().await.map_err(|e| format!("JSON: {}", e))?;
|
||||||
|
// Pass through the relevant fields (same structure as JS version)
|
||||||
|
Ok(json!({
|
||||||
|
"domain": {
|
||||||
|
"name": data["web"]["domain"],
|
||||||
|
"original": data["web"]["originalDomain"],
|
||||||
|
"ipv6_support": data["web"]["ipV6Support"],
|
||||||
|
},
|
||||||
|
"web": {
|
||||||
|
"ips": data["web"]["lookups"],
|
||||||
|
"providers": data["web"]["providers"],
|
||||||
|
},
|
||||||
|
"nameserver": data["nameserver"],
|
||||||
|
"mail": {"incoming": data["incomingMail"], "outgoing": data["outgoingMail"]},
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Harga Pangan (BAPANAS) — Indonesian government food prices
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const BAPANAS_KEY: &str = "zHWbt7U2qvPoUDkiUgvnOqYrtj3zClR7unnH2G4apE7HcMV4QyNC6BSD0yV3uvSHqS91TxwE8aMDTiCznmGceEX3zQmO1Xwq7TJblotIt2CpwvK6YjRKDJwcgMJwav9p4RshM3nfuFyurSQQv9BhueMJ0HJ778oD";
|
||||||
|
|
||||||
|
pub async fn fetch_hargapangan() -> Result<Value, String> {
|
||||||
|
let url = "https://api-panelhargav2.badanpangan.go.id/api/front/harga-pangan-informasi";
|
||||||
|
let resp = http_client()
|
||||||
|
.client()
|
||||||
|
.get(url)
|
||||||
|
.header(USER_AGENT, CHROME_UA)
|
||||||
|
.header("accept", "application/json")
|
||||||
|
.header("x-api-key", BAPANAS_KEY)
|
||||||
|
.header("origin", "https://panelharga.badanpangan.go.id")
|
||||||
|
.header("referer", "https://panelharga.badanpangan.go.id/")
|
||||||
|
.query(&[("level_harga_id", "1")])
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP: {}", e))?;
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(format!("HTTP {}", resp.status()));
|
||||||
|
}
|
||||||
|
let data: Value = resp.json().await.map_err(|e| format!("JSON: {}", e))?;
|
||||||
|
if data.get("status").and_then(|s| s.as_str()) == Some("success") {
|
||||||
|
Ok(data.get("data").cloned().unwrap_or(Value::Null))
|
||||||
|
} else {
|
||||||
|
Ok(Value::Null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Cek Resi (cekresi.com) — Indonesian package tracking
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
fn aes_encrypt_resi(resi: &str) -> String {
|
||||||
|
use aes::cipher::generic_array::GenericArray;
|
||||||
|
use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit};
|
||||||
|
type Aes128CbcEnc = cbc::Encryptor<aes::Aes128>;
|
||||||
|
let key_hex = "79540e250fdb16afac03e19c46dbdeb3";
|
||||||
|
let iv_hex = "eb2bb9425e81ffa942522e4414e95bd0";
|
||||||
|
let key_bytes = hex::decode(key_hex).expect("bad key hex");
|
||||||
|
let iv_bytes = hex::decode(iv_hex).expect("bad iv hex");
|
||||||
|
let key = GenericArray::from_slice(&key_bytes);
|
||||||
|
let iv = GenericArray::from_slice(&iv_bytes);
|
||||||
|
|
||||||
|
let data = resi.as_bytes();
|
||||||
|
let mut buf = vec![0u8; data.len() + 16];
|
||||||
|
buf[..data.len()].copy_from_slice(data);
|
||||||
|
let ct = Aes128CbcEnc::new(key, iv)
|
||||||
|
.encrypt_padded_mut::<Pkcs7>(&mut buf, data.len())
|
||||||
|
.expect("encrypt");
|
||||||
|
base64::Engine::encode(&base64::engine::general_purpose::STANDARD, ct)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn detect_ekspedisi(resi: &str) -> Option<&'static str> {
|
||||||
|
let upper = resi.to_uppercase();
|
||||||
|
let map = [
|
||||||
|
("jne", "JNE"),
|
||||||
|
("pos", "POS"),
|
||||||
|
("tiki", "TIKI"),
|
||||||
|
("jnt", "J&T"),
|
||||||
|
("sicepat", "SICEPAT"),
|
||||||
|
("anteraja", "ANTERAJA"),
|
||||||
|
("ninja", "NINJA"),
|
||||||
|
("lionparcel", "LIONPARCEL"),
|
||||||
|
];
|
||||||
|
for (prefix, code) in map {
|
||||||
|
if upper.starts_with(code) {
|
||||||
|
return Some(prefix);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn fetch_cek_resi(noresi: &str, ekspedisi: Option<&str>) -> Result<Value, String> {
|
||||||
|
let noresi = noresi.trim().to_uppercase();
|
||||||
|
let ekspedisi = match ekspedisi {
|
||||||
|
Some(e) => e.to_string(),
|
||||||
|
None => detect_ekspedisi(&noresi)
|
||||||
|
.ok_or_else(|| "Ekspedisi tidak ditemukan dari nomor resi".to_string())?
|
||||||
|
.to_string(),
|
||||||
|
};
|
||||||
|
let timers = aes_encrypt_resi(&noresi);
|
||||||
|
|
||||||
|
// 1. GET the form to extract viewstate + secret_key
|
||||||
|
let client = http_client().client();
|
||||||
|
let html = client
|
||||||
|
.get("https://cekresi.com/")
|
||||||
|
.header(USER_AGENT, CHROME_UA)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("HTTP: {}", e))?
|
||||||
|
.text()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("Body: {}", e))?;
|
||||||
|
let (viewstate, secret_key) = {
|
||||||
|
let doc = Html::parse_document(&html);
|
||||||
|
let input_sel = Selector::parse(r#"input[name="viewstate"]"#).unwrap();
|
||||||
|
let secret_sel = Selector::parse(r#"input[name="secret_key"]"#).unwrap();
|
||||||
|
let vs = doc
|
||||||
|
.select(&input_sel)
|
||||||
|
.next()
|
||||||
|
.and_then(|e| e.value().attr("value"))
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
let sk = doc
|
||||||
|
.select(&secret_sel)
|
||||||
|
.next()
|
||||||
|
.and_then(|e| e.value().attr("value"))
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
(vs, sk)
|
||||||
|
};
|
||||||
|
|
||||||
|
// 2. POST the tracking request
|
||||||
|
let w = &format!("{:x}", fastrand::u32(..));
|
||||||
|
let post_url = format!(
|
||||||
|
"https://apa2.cekresi.com/cekresi/resi/initialize.php?ui=e0ad7e971ce77822056ba7a155f85c11&p=1&w={}",
|
||||||
|
w
|
||||||
|
);
|
||||||
|
let form = [
|
||||||
|
("viewstate", viewstate.as_str()),
|
||||||
|
("secret_key", secret_key.as_str()),
|
||||||
|
("e", &ekspedisi),
|
||||||
|
("noresi", &noresi),
|
||||||
|
("timers", &timers),
|
||||||
|
];
|
||||||
|
let resp = client.post(&post_url)
|
||||||
|
.header(USER_AGENT, "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Mobile Safari/537.36")
|
||||||
|
.header("referer", "https://cekresi.com/")
|
||||||
|
.header("origin", "https://cekresi.com")
|
||||||
|
.form(&form)
|
||||||
|
.send().await.map_err(|e| format!("HTTP: {}", e))?;
|
||||||
|
let body = resp.text().await.map_err(|e| format!("Body: {}", e))?;
|
||||||
|
let doc2 = Html::parse_document(&body);
|
||||||
|
|
||||||
|
// 3. Parse results
|
||||||
|
let alert_sel = Selector::parse(".alert.alert-success").unwrap();
|
||||||
|
let success = doc2.select(&alert_sel).next().is_some();
|
||||||
|
|
||||||
|
if success {
|
||||||
|
let message = doc2
|
||||||
|
.select(&alert_sel)
|
||||||
|
.next()
|
||||||
|
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let table_sel = Selector::parse("table.table-striped tbody tr").unwrap();
|
||||||
|
let mut status = String::new();
|
||||||
|
let mut tanggal = String::new();
|
||||||
|
for row in doc2.select(&table_sel) {
|
||||||
|
let tds: Vec<_> = row.select(&Selector::parse("td").unwrap()).collect();
|
||||||
|
if tds.len() >= 3 {
|
||||||
|
let label = tds[0].text().collect::<String>().trim().to_string();
|
||||||
|
let value = tds[2].text().collect::<String>().trim().to_string();
|
||||||
|
if label == "Tanggal Pengiriman" {
|
||||||
|
tanggal = value;
|
||||||
|
} else if label == "Status" {
|
||||||
|
status = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let last_pos = Selector::parse("#last_position").unwrap();
|
||||||
|
let last_position = doc2
|
||||||
|
.select(&last_pos)
|
||||||
|
.next()
|
||||||
|
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// History: scan all tables for the one whose rows have exactly 2 cells
|
||||||
|
// (tanggal, keterangan). The status table above uses 3 cells, so the
|
||||||
|
// 2-cell table after the "History" heading is unambiguous.
|
||||||
|
let mut history: Vec<Value> = Vec::new();
|
||||||
|
let table_sel_all = Selector::parse("table").unwrap();
|
||||||
|
let td_sel = Selector::parse("td").unwrap();
|
||||||
|
for table in doc2.select(&table_sel_all) {
|
||||||
|
let mut rows: Vec<Value> = Vec::new();
|
||||||
|
for tr in table.select(&Selector::parse("tbody tr").unwrap()) {
|
||||||
|
let tds: Vec<_> = tr.select(&td_sel).collect();
|
||||||
|
if tds.len() == 2 {
|
||||||
|
rows.push(json!({
|
||||||
|
"tanggal": tds[0].text().collect::<String>().trim(),
|
||||||
|
"keterangan": tds[1].text().collect::<String>().trim(),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A table with 2-cell rows is the history table; skip its header row.
|
||||||
|
if rows.len() > 1 {
|
||||||
|
history = rows.into_iter().skip(1).collect();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(json!({
|
||||||
|
"success": true,
|
||||||
|
"message": message,
|
||||||
|
"data": {
|
||||||
|
"resi": noresi,
|
||||||
|
"ekspedisi": doc2.select(&Selector::parse("#nama_expedisi").unwrap()).next()
|
||||||
|
.map(|e| e.text().collect::<String>().trim().to_string()).unwrap_or_default(),
|
||||||
|
"status": status,
|
||||||
|
"tanggalKirim": tanggal,
|
||||||
|
"lastPosition": last_position,
|
||||||
|
"history": history,
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
} else {
|
||||||
|
let alert_err = Selector::parse(".alert.alert-danger, .alert.alert-warning").unwrap();
|
||||||
|
let message = doc2
|
||||||
|
.select(&alert_err)
|
||||||
|
.next()
|
||||||
|
.map(|e| e.text().collect::<String>().trim().to_string())
|
||||||
|
.unwrap_or_else(|| "Tidak dapat mengambil informasi resi".to_string());
|
||||||
|
Ok(json!({"success": false, "message": message}))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
//! Infrastructure — Weebs endpoints (anime/manga info, waifu pics, whatanime).
|
|
||||||
//!
|
|
||||||
//! Ported from Shirokami-API `scraper/weebs/*.js`. These are public JSON
|
|
||||||
//! APIs that we proxy/forward (jikan.moe, waifu.pics, trace.moe).
|
|
||||||
|
|
||||||
use crate::infrastructure::utils::http_client::http_client;
|
|
||||||
use reqwest::header::USER_AGENT;
|
|
||||||
use serde_json::{json, Value};
|
|
||||||
|
|
||||||
/// Anime info from jikan.moe, ported from anime-search's `getAnimeInfo`.
|
|
||||||
pub async fn fetch_anime_info(query: &str) -> Result<Value, String> {
|
|
||||||
let url = format!("https://api.jikan.moe/v4/anime?q={}", urlencode(query));
|
|
||||||
let data = get_json(&url).await?;
|
|
||||||
|
|
||||||
let arr = data
|
|
||||||
.get("data")
|
|
||||||
.and_then(|d| d.as_array())
|
|
||||||
.cloned()
|
|
||||||
.unwrap_or_default();
|
|
||||||
if arr.is_empty() {
|
|
||||||
return Ok(json!({ "error": "Anime tidak ditemukan" }));
|
|
||||||
}
|
|
||||||
let anime = &arr[0];
|
|
||||||
|
|
||||||
let judul = anime
|
|
||||||
.get("titles")
|
|
||||||
.and_then(|t| t.as_array())
|
|
||||||
.and_then(|t| t.first())
|
|
||||||
.and_then(|t| t.get("title"))
|
|
||||||
.and_then(|t| t.as_str())
|
|
||||||
.unwrap_or("Title not available");
|
|
||||||
let genrenya = join_names(anime.get("genres"));
|
|
||||||
|
|
||||||
Ok(json!({
|
|
||||||
"title": judul,
|
|
||||||
"url": anime.get("url").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"type": anime.get("type").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"score": anime.get("score").cloned().unwrap_or(Value::Null),
|
|
||||||
"members": anime.get("members").cloned().unwrap_or(Value::Null),
|
|
||||||
"status": anime.get("status").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"synopsis": anime.get("synopsis").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"favorites": anime.get("favorites").cloned().unwrap_or(Value::Null),
|
|
||||||
"images": anime.get("images").cloned().unwrap_or(Value::Null),
|
|
||||||
"genres": genrenya,
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Manga info from jikan.moe, ported from anime-search's `getMangaInfo`.
|
|
||||||
pub async fn fetch_manga_info(query: &str) -> Result<Value, String> {
|
|
||||||
let url = format!("https://api.jikan.moe/v4/manga?q={}", urlencode(query));
|
|
||||||
let data = get_json(&url).await?;
|
|
||||||
|
|
||||||
let arr = data
|
|
||||||
.get("data")
|
|
||||||
.and_then(|d| d.as_array())
|
|
||||||
.cloned()
|
|
||||||
.unwrap_or_default();
|
|
||||||
if arr.is_empty() {
|
|
||||||
return Ok(json!({ "error": "Manga tidak ditemukan" }));
|
|
||||||
}
|
|
||||||
let manga = &arr[0];
|
|
||||||
|
|
||||||
let judul = manga
|
|
||||||
.get("titles")
|
|
||||||
.and_then(|t| t.as_array())
|
|
||||||
.map(|tt| {
|
|
||||||
tt.iter()
|
|
||||||
.filter_map(|j| {
|
|
||||||
let title = j.get("title").and_then(|v| v.as_str()).unwrap_or("");
|
|
||||||
let ty = j.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
|
||||||
if title.is_empty() {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
Some(format!("{} [{}]", title, ty))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join(", ")
|
|
||||||
})
|
|
||||||
.unwrap_or_default();
|
|
||||||
let genrenya = join_names(manga.get("genres"));
|
|
||||||
|
|
||||||
Ok(json!({
|
|
||||||
"title": judul,
|
|
||||||
"chapters": manga.get("chapters").cloned().unwrap_or(Value::Null),
|
|
||||||
"type": manga.get("type").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"status": manga.get("status").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"genre": genrenya,
|
|
||||||
"volumes": manga.get("volumes").cloned().unwrap_or(Value::Null),
|
|
||||||
"favorites": manga.get("favorites").cloned().unwrap_or(Value::Null),
|
|
||||||
"score": manga.get("score").cloned().unwrap_or(Value::Null),
|
|
||||||
"scored": manga.get("scored").cloned().unwrap_or(Value::Null),
|
|
||||||
"scored_by": manga.get("scored_by").cloned().unwrap_or(Value::Null),
|
|
||||||
"rank": manga.get("rank").cloned().unwrap_or(Value::Null),
|
|
||||||
"popularity": manga.get("popularity").cloned().unwrap_or(Value::Null),
|
|
||||||
"members": manga.get("members").cloned().unwrap_or(Value::Null),
|
|
||||||
"url": manga.get("url").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"background": manga.get("background").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"synopsis": manga.get("synopsis").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// What anime is in this screenshot? trace.moe reverse-image search.
|
|
||||||
pub async fn fetch_whatanime(url: &str) -> Result<Value, String> {
|
|
||||||
let api = format!(
|
|
||||||
"https://api.trace.moe/search?cutBorders&url={}",
|
|
||||||
urlencode(url)
|
|
||||||
);
|
|
||||||
let data = get_json(&api).await?;
|
|
||||||
|
|
||||||
let result = data
|
|
||||||
.get("result")
|
|
||||||
.and_then(|r| r.as_array())
|
|
||||||
.and_then(|r| r.first())
|
|
||||||
.cloned();
|
|
||||||
let Some(r) = result else {
|
|
||||||
return Ok(json!({ "error": "Anime tidak ditemukan" }));
|
|
||||||
};
|
|
||||||
|
|
||||||
let similarity = r
|
|
||||||
.get("similarity")
|
|
||||||
.and_then(|s| s.as_f64())
|
|
||||||
.map(|s| (s * 100.0).round() as i64)
|
|
||||||
.unwrap_or(0);
|
|
||||||
|
|
||||||
Ok(json!({
|
|
||||||
"judul": r.get("filename").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"episode": r.get("episode").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"similarity": similarity,
|
|
||||||
"videoURL": r.get("video").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
"videoIMG": r.get("image").and_then(|v| v.as_str()).unwrap_or(""),
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// SFW waifu image from waifu.pics.
|
|
||||||
pub async fn fetch_sfw_waifu(tag: &str) -> Result<Value, String> {
|
|
||||||
let url = format!("https://api.waifu.pics/sfw/{}", urlencode(tag));
|
|
||||||
get_json(&url).await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// NSFW waifu image from waifu.pics.
|
|
||||||
pub async fn fetch_nsfw_waifu(tag: &str) -> Result<Value, String> {
|
|
||||||
let url = format!("https://api.waifu.pics/nsfw/{}", urlencode(tag));
|
|
||||||
get_json(&url).await
|
|
||||||
}
|
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Helpers
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
async fn get_json(url: &str) -> Result<Value, String> {
|
|
||||||
let resp = http_client()
|
|
||||||
.client()
|
|
||||||
.get(url)
|
|
||||||
.header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("HTTP error: {}", e))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
return Err(format!("HTTP {} from upstream", resp.status()));
|
|
||||||
}
|
|
||||||
resp.json::<Value>()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("JSON parse: {}", e))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Join an array of { name: "X" } objects into a comma-separated string.
|
|
||||||
fn join_names(v: Option<&Value>) -> String {
|
|
||||||
v.and_then(|g| g.as_array())
|
|
||||||
.map(|arr| {
|
|
||||||
arr.iter()
|
|
||||||
.filter_map(|g| g.get("name").and_then(|n| n.as_str()).map(|s| s.to_string()))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join(", ")
|
|
||||||
})
|
|
||||||
.unwrap_or_default()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Minimal URL-encode (space -> %20, keep alphanumerics and safe chars).
|
|
||||||
fn urlencode(s: &str) -> String {
|
|
||||||
let mut out = String::new();
|
|
||||||
for b in s.bytes() {
|
|
||||||
match b {
|
|
||||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
|
||||||
out.push(b as char)
|
|
||||||
}
|
|
||||||
b' ' => out.push_str("%20"),
|
|
||||||
_ => out.push_str(&format!("%{:02X}", b)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out
|
|
||||||
}
|
|
||||||
@@ -4,7 +4,7 @@ use utoipa::OpenApi;
|
|||||||
#[derive(OpenApi)]
|
#[derive(OpenApi)]
|
||||||
#[openapi(
|
#[openapi(
|
||||||
paths(
|
paths(
|
||||||
// Anime module handlers
|
// ── Anime module ──────────────────────────────────────────
|
||||||
crate::presentation::handler::anime::anime_index,
|
crate::presentation::handler::anime::anime_index,
|
||||||
crate::presentation::handler::anime::genres,
|
crate::presentation::handler::anime::genres,
|
||||||
crate::presentation::handler::anime::detail_slug,
|
crate::presentation::handler::anime::detail_slug,
|
||||||
@@ -16,7 +16,7 @@ use utoipa::OpenApi;
|
|||||||
crate::presentation::handler::anime::search_slug_page,
|
crate::presentation::handler::anime::search_slug_page,
|
||||||
crate::presentation::handler::anime::genre_slug_index,
|
crate::presentation::handler::anime::genre_slug_index,
|
||||||
crate::presentation::handler::anime::genre_slug_page,
|
crate::presentation::handler::anime::genre_slug_page,
|
||||||
// Anime2 module handlers
|
// ── Anime2 module ─────────────────────────────────────────
|
||||||
crate::presentation::handler::anime2::index,
|
crate::presentation::handler::anime2::index,
|
||||||
crate::presentation::handler::anime2::genre_list,
|
crate::presentation::handler::anime2::genre_list,
|
||||||
crate::presentation::handler::anime2::filter,
|
crate::presentation::handler::anime2::filter,
|
||||||
@@ -28,7 +28,7 @@ use utoipa::OpenApi;
|
|||||||
crate::presentation::handler::anime2::latest_slug,
|
crate::presentation::handler::anime2::latest_slug,
|
||||||
crate::presentation::handler::anime2::ongoing_anime_slug,
|
crate::presentation::handler::anime2::ongoing_anime_slug,
|
||||||
crate::presentation::handler::anime2::complete_anime_slug,
|
crate::presentation::handler::anime2::complete_anime_slug,
|
||||||
// Komik module handlers
|
// ── Komik module ──────────────────────────────────────────
|
||||||
crate::presentation::handler::komik::genre_list,
|
crate::presentation::handler::komik::genre_list,
|
||||||
crate::presentation::handler::komik::chapter_slug,
|
crate::presentation::handler::komik::chapter_slug,
|
||||||
crate::presentation::handler::komik::detail_slug,
|
crate::presentation::handler::komik::detail_slug,
|
||||||
@@ -40,19 +40,75 @@ use utoipa::OpenApi;
|
|||||||
crate::presentation::handler::komik::popular_slug,
|
crate::presentation::handler::komik::popular_slug,
|
||||||
crate::presentation::handler::komik::search_slug,
|
crate::presentation::handler::komik::search_slug,
|
||||||
crate::presentation::handler::komik::search_slug_page,
|
crate::presentation::handler::komik::search_slug_page,
|
||||||
// Proxy module handlers
|
// ── Download module ────────────────────────────────────────
|
||||||
|
crate::presentation::handler::downloader::download,
|
||||||
|
crate::presentation::handler::downloader::detect_platform_handler,
|
||||||
|
crate::presentation::handler::downloader::download_instagram,
|
||||||
|
crate::presentation::handler::downloader::download_facebook,
|
||||||
|
crate::presentation::handler::downloader::download_tiktok,
|
||||||
|
crate::presentation::handler::downloader::download_youtube,
|
||||||
|
crate::presentation::handler::downloader::download_youtube_mp3,
|
||||||
|
crate::presentation::handler::downloader::download_spotify,
|
||||||
|
crate::presentation::handler::downloader::download_twitter,
|
||||||
|
crate::presentation::handler::downloader::download_pinterest,
|
||||||
|
crate::presentation::handler::downloader::download_mega,
|
||||||
|
crate::presentation::handler::downloader::download_terabox,
|
||||||
|
crate::presentation::handler::downloader::download_gdrive,
|
||||||
|
crate::presentation::handler::downloader::download_mediafire,
|
||||||
|
crate::presentation::handler::downloader::download_pixeldrain,
|
||||||
|
crate::presentation::handler::downloader::download_threads,
|
||||||
|
crate::presentation::handler::downloader::download_doodstream,
|
||||||
|
crate::presentation::handler::downloader::download_krakenfiles,
|
||||||
|
crate::presentation::handler::downloader::download_danbooru,
|
||||||
|
crate::presentation::handler::downloader::download_soundcloud,
|
||||||
|
crate::presentation::handler::downloader::download_dailymotion,
|
||||||
|
crate::presentation::handler::downloader::download_reddit,
|
||||||
|
crate::presentation::handler::downloader::download_streamable,
|
||||||
|
crate::presentation::handler::downloader::download_videy,
|
||||||
|
crate::presentation::handler::downloader::download_bilibili,
|
||||||
|
crate::presentation::handler::downloader::serve_merged_file,
|
||||||
|
// ── Misc module ────────────────────────────────────────────
|
||||||
|
crate::presentation::handler::misc::currency_converter_handler,
|
||||||
|
crate::presentation::handler::misc::harga_emas_handler,
|
||||||
|
crate::presentation::handler::misc::kurs_bca_handler,
|
||||||
|
crate::presentation::handler::misc::server_info_handler,
|
||||||
|
// ── Stalk module ───────────────────────────────────────────
|
||||||
|
crate::presentation::handler::stalk::github_handler,
|
||||||
|
crate::presentation::handler::stalk::youtube_handler,
|
||||||
|
crate::presentation::handler::stalk::twitter_handler,
|
||||||
|
// ── Search module ──────────────────────────────────────────
|
||||||
|
crate::presentation::handler::search::bmkg_handler,
|
||||||
|
crate::presentation::handler::search::jadwal_sholat_handler,
|
||||||
|
crate::presentation::handler::search::weather_handler,
|
||||||
|
crate::presentation::handler::search::google_handler,
|
||||||
|
crate::presentation::handler::search::yt_handler,
|
||||||
|
// ── Tool module ────────────────────────────────────────────
|
||||||
|
crate::presentation::handler::tools::whois_handler,
|
||||||
|
crate::presentation::handler::tools::ip_location_handler,
|
||||||
|
crate::presentation::handler::tools::tinyurl_handler,
|
||||||
|
crate::presentation::handler::tools::check_hosting_handler,
|
||||||
|
crate::presentation::handler::tools::hargapangan_handler,
|
||||||
|
crate::presentation::handler::tools::cek_resi_handler,
|
||||||
|
// ── Image module ───────────────────────────────────────────
|
||||||
|
crate::presentation::handler::image::brat_handler,
|
||||||
|
crate::presentation::handler::image::brat_animated_handler,
|
||||||
),
|
),
|
||||||
components(
|
components(
|
||||||
schemas(
|
schemas(
|
||||||
// Application response wrapper
|
|
||||||
crate::presentation::dto::common::ApiResponse<String>,
|
crate::presentation::dto::common::ApiResponse<String>,
|
||||||
|
crate::presentation::dto::downloader::DownloadResponse,
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
tags(
|
tags(
|
||||||
(name = "anime", description = "Anime endpoints"),
|
(name = "anime", description = "Anime endpoints"),
|
||||||
(name = "anime2", description = "Anime2 endpoints"),
|
(name = "anime2", description = "Anime2 endpoints"),
|
||||||
(name = "komik", description = "Komik endpoints"),
|
(name = "komik", description = "Komik endpoints"),
|
||||||
(name = "proxy", description = "Proxy endpoints"),
|
(name = "download", description = "Media download endpoints (23 platforms)"),
|
||||||
|
(name = "misc", description = "Miscellaneous utilities (currency, gold, server)"),
|
||||||
|
(name = "stalk", description = "User profile stalking (GitHub, YT, Twitter)"),
|
||||||
|
(name = "search", description = "Search endpoints (BMKG, prayer times, weather, YouTube, Google)"),
|
||||||
|
(name = "tool", description = "Utility tools (WHOIS, IP lookup, URL shortener, tracking, hosting)"),
|
||||||
|
(name = "image", description = "Image generation (brat static & animated)"),
|
||||||
)
|
)
|
||||||
)]
|
)]
|
||||||
pub struct ModuleApiDoc;
|
pub struct ModuleApiDoc;
|
||||||
@@ -7,40 +7,85 @@
|
|||||||
use axum::extract::Query;
|
use axum::extract::Query;
|
||||||
use axum::Json;
|
use axum::Json;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
use serde_json::Value;
|
||||||
|
use utoipa::IntoParams;
|
||||||
|
|
||||||
use crate::application::downloader as use_cases;
|
use crate::application::downloader as use_cases;
|
||||||
use crate::presentation::dto::downloader::DownloadResponse;
|
use crate::presentation::dto::downloader::DownloadResponse;
|
||||||
use crate::presentation::error::AppError;
|
use crate::presentation::error::AppError;
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// DownloadParams + response
|
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
/// Request params for downloader endpoints.
|
/// Request params for downloader endpoints.
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize, IntoParams)]
|
||||||
pub struct DownloadParams {
|
pub struct DownloadParams {
|
||||||
pub url: String,
|
pub url: String,
|
||||||
pub cookies: Option<String>,
|
pub cookies: Option<String>,
|
||||||
pub api_key: Option<String>,
|
pub api_key: Option<String>,
|
||||||
pub quality: Option<String>,
|
pub quality: Option<String>,
|
||||||
|
/// When `true`, YouTube downloads are merged server-side (video + audio
|
||||||
|
/// into a single MP4) instead of returning separate video/audio URLs.
|
||||||
|
pub merge: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
// ========================================================================
|
||||||
// Handlers
|
// Handlers
|
||||||
// ============================================================================
|
// ========================================================================
|
||||||
|
|
||||||
/// All-in-one downloader with auto-detection.
|
/// All-in-one auto-detect downloader.
|
||||||
|
/// `/download?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download(
|
pub async fn download(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
let result = use_cases::download_all_in_one(¶ms.url, params.cookies.as_deref()).await?;
|
let merge = params
|
||||||
|
.merge
|
||||||
|
.as_deref()
|
||||||
|
.map(|v| v == "true" || v == "1")
|
||||||
|
.unwrap_or(false);
|
||||||
|
let result = if merge {
|
||||||
|
// All-in-one with merge=true → force merge for YouTube URLs.
|
||||||
|
if crate::application::downloader::detect_platform(¶ms.url) == "youtube" {
|
||||||
|
use_cases::download_youtube_merge(
|
||||||
|
¶ms.url,
|
||||||
|
params.quality.as_deref().unwrap_or("720"),
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
} else {
|
||||||
|
use_cases::download_all_in_one(¶ms.url, params.cookies.as_deref()).await?
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
use_cases::download_all_in_one(¶ms.url, params.cookies.as_deref()).await?
|
||||||
|
};
|
||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Platform detection endpoint.
|
/// Detect platform from URL.
|
||||||
|
/// `/detect-platform?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/detect-platform",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_detect_platform_handler",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn detect_platform_handler(
|
pub async fn detect_platform_handler(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<serde_json::Value>, AppError> {
|
) -> Result<Json<Value>, AppError> {
|
||||||
let platform = use_cases::detect_platform(¶ms.url);
|
let platform = use_cases::detect_platform(¶ms.url);
|
||||||
Ok(Json(serde_json::json!({
|
Ok(Json(serde_json::json!({
|
||||||
"platform": platform,
|
"platform": platform,
|
||||||
@@ -48,8 +93,20 @@ pub async fn detect_platform_handler(
|
|||||||
})))
|
})))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Generate handler for each downloader platform.
|
/// Download Instagram media.
|
||||||
/// Handler for _instagram.
|
/// `/download/instagram?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/instagram",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_instagram",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_instagram(
|
pub async fn download_instagram(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -57,7 +114,20 @@ pub async fn download_instagram(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _facebook.
|
/// Download Facebook video.
|
||||||
|
/// `/download/facebook?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/facebook",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_facebook",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_facebook(
|
pub async fn download_facebook(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -65,7 +135,20 @@ pub async fn download_facebook(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _tiktok.
|
/// Download TikTok video.
|
||||||
|
/// `/download/tiktok?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/tiktok",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_tiktok",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_tiktok(
|
pub async fn download_tiktok(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -73,17 +156,51 @@ pub async fn download_tiktok(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _youtube.
|
/// Download YouTube video.
|
||||||
|
/// `/download/youtube?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/youtube",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_youtube",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_youtube(
|
pub async fn download_youtube(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
let result =
|
let quality = params.quality.as_deref().unwrap_or("720");
|
||||||
use_cases::download_youtube(¶ms.url, params.quality.as_deref().unwrap_or("720"))
|
let merge = params
|
||||||
.await?;
|
.merge
|
||||||
|
.as_deref()
|
||||||
|
.map(|v| v == "true" || v == "1")
|
||||||
|
.unwrap_or(false);
|
||||||
|
let result = if merge {
|
||||||
|
use_cases::download_youtube_merge(¶ms.url, quality).await?
|
||||||
|
} else {
|
||||||
|
use_cases::download_youtube(¶ms.url, quality).await?
|
||||||
|
};
|
||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _youtube_mp3.
|
/// Convert YouTube to MP3.
|
||||||
|
/// `/download/youtube/mp3?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/youtube/mp3",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_youtube_mp3",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_youtube_mp3(
|
pub async fn download_youtube_mp3(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -91,7 +208,20 @@ pub async fn download_youtube_mp3(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _spotify.
|
/// Download Spotify track.
|
||||||
|
/// `/download/spotify?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/spotify",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_spotify",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_spotify(
|
pub async fn download_spotify(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -99,7 +229,20 @@ pub async fn download_spotify(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _twitter.
|
/// Download Twitter/X media.
|
||||||
|
/// `/download/twitter?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/twitter",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_twitter",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_twitter(
|
pub async fn download_twitter(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -107,7 +250,20 @@ pub async fn download_twitter(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _pinterest.
|
/// Download Pinterest pin.
|
||||||
|
/// `/download/pinterest?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/pinterest",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_pinterest",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_pinterest(
|
pub async fn download_pinterest(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -115,7 +271,20 @@ pub async fn download_pinterest(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _mega.
|
/// Download from Mega.
|
||||||
|
/// `/download/mega?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/mega",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_mega",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_mega(
|
pub async fn download_mega(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -123,7 +292,20 @@ pub async fn download_mega(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _terabox.
|
/// Download from Terabox.
|
||||||
|
/// `/download/terabox?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/terabox",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_terabox",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_terabox(
|
pub async fn download_terabox(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -131,7 +313,115 @@ pub async fn download_terabox(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _gdrive.
|
/// Stream a TeraBox file through this API.
|
||||||
|
/// `/proxy/terabox?surl=...`
|
||||||
|
///
|
||||||
|
/// The resolver's `/download` endpoint replays the full 302 chain WITH
|
||||||
|
/// cookies and streams real bytes — this handler forwards those bytes to the
|
||||||
|
/// caller so the returned URL is a public, downloadable link (not a 403
|
||||||
|
/// dlink, not a private 127.0.0.1 URL).
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/proxy/terabox",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_proxy_terabox",
|
||||||
|
params(
|
||||||
|
("surl" = String, Query, description = "TeraBox share short URL ID"),
|
||||||
|
("filename" = Option<String>, Query, description = "Download file name hint"),
|
||||||
|
),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "File stream"),
|
||||||
|
(status = 400, description = "Missing surl"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn proxy_terabox(
|
||||||
|
Query(params): Query<TeraboxProxyParams>,
|
||||||
|
) -> Result<axum::response::Response, AppError> {
|
||||||
|
let surl = params.surl;
|
||||||
|
if surl.is_empty() {
|
||||||
|
return Err(AppError::BadRequest("missing surl parameter".into()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let resolver_url = std::env::var("TERABOX_RESOLVER_URL")
|
||||||
|
.unwrap_or_else(|_| "http://127.0.0.1:4092".to_string());
|
||||||
|
let fetch_url = format!("{}/download?surl={}", resolver_url, surl);
|
||||||
|
|
||||||
|
// Ask the resolver for the redirect chain (it follows 302s internally with
|
||||||
|
// cookies) — but we only want the final CDN stream URL. The resolver
|
||||||
|
// already streams; simplest reliable path is to just proxy the resolver's
|
||||||
|
// response body as-is.
|
||||||
|
let client = crate::infrastructure::utils::http_client::http_client();
|
||||||
|
|
||||||
|
let resp = client
|
||||||
|
.client()
|
||||||
|
.get(&fetch_url)
|
||||||
|
.header("User-Agent", "Mozilla/5.0")
|
||||||
|
.timeout(std::time::Duration::from_secs(120))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| AppError::ScraperError(format!("terabox proxy fetch failed: {e}")))?;
|
||||||
|
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
let status = resp.status();
|
||||||
|
let body = resp.text().await.unwrap_or_default();
|
||||||
|
return Err(AppError::ScraperError(format!(
|
||||||
|
"terabox proxy returned HTTP {status}: {body}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let content_type = resp
|
||||||
|
.headers()
|
||||||
|
.get(reqwest::header::CONTENT_TYPE)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("application/octet-stream")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let content_disposition = resp
|
||||||
|
.headers()
|
||||||
|
.get(reqwest::header::CONTENT_DISPOSITION)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.unwrap_or_else(|| {
|
||||||
|
let fname = params
|
||||||
|
.filename
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.unwrap_or_else(|| "terabox-download.bin".to_string());
|
||||||
|
format!("attachment; filename=\"{fname}\"")
|
||||||
|
});
|
||||||
|
|
||||||
|
let stream = resp.bytes_stream();
|
||||||
|
|
||||||
|
Ok(axum::response::Response::builder()
|
||||||
|
.header("Content-Type", content_type)
|
||||||
|
.header("Content-Disposition", content_disposition)
|
||||||
|
.header("X-Accel-Buffering", "no")
|
||||||
|
.header("Cache-Control", "no-store")
|
||||||
|
.body(axum::body::Body::from_stream(stream))
|
||||||
|
.map_err(|e| AppError::Internal(format!("build stream response: {e}")))?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Params for the TeraBox streaming proxy.
|
||||||
|
#[derive(Debug, Deserialize, IntoParams)]
|
||||||
|
pub struct TeraboxProxyParams {
|
||||||
|
pub surl: String,
|
||||||
|
pub filename: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Download from Google Drive.
|
||||||
|
/// `/download/gdrive?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/gdrive",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_gdrive",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_gdrive(
|
pub async fn download_gdrive(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -139,7 +429,20 @@ pub async fn download_gdrive(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _mediafire.
|
/// Download from MediaFire.
|
||||||
|
/// `/download/mediafire?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/mediafire",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_mediafire",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_mediafire(
|
pub async fn download_mediafire(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -147,7 +450,20 @@ pub async fn download_mediafire(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _pixeldrain.
|
/// Download from Pixeldrain.
|
||||||
|
/// `/download/pixeldrain?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/pixeldrain",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_pixeldrain",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_pixeldrain(
|
pub async fn download_pixeldrain(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -155,7 +471,20 @@ pub async fn download_pixeldrain(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _threads.
|
/// Download Threads media.
|
||||||
|
/// `/download/threads?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/threads",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_threads",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_threads(
|
pub async fn download_threads(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -163,7 +492,20 @@ pub async fn download_threads(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _doodstream.
|
/// Download from DoodStream.
|
||||||
|
/// `/download/doodstream?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/doodstream",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_doodstream",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_doodstream(
|
pub async fn download_doodstream(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -171,7 +513,20 @@ pub async fn download_doodstream(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _krakenfiles.
|
/// Download from KrakenFiles.
|
||||||
|
/// `/download/krakenfiles?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/krakenfiles",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_krakenfiles",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_krakenfiles(
|
pub async fn download_krakenfiles(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -179,7 +534,20 @@ pub async fn download_krakenfiles(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _danbooru.
|
/// Download from Danbooru.
|
||||||
|
/// `/download/danbooru?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/danbooru",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_danbooru",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_danbooru(
|
pub async fn download_danbooru(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -187,7 +555,20 @@ pub async fn download_danbooru(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _soundcloud.
|
/// Download SoundCloud track.
|
||||||
|
/// `/download/soundcloud?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/soundcloud",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_soundcloud",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_soundcloud(
|
pub async fn download_soundcloud(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -195,7 +576,20 @@ pub async fn download_soundcloud(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _dailymotion.
|
/// Download Dailymotion video.
|
||||||
|
/// `/download/dailymotion?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/dailymotion",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_dailymotion",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_dailymotion(
|
pub async fn download_dailymotion(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -203,7 +597,20 @@ pub async fn download_dailymotion(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _reddit.
|
/// Download Reddit media.
|
||||||
|
/// `/download/reddit?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/reddit",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_reddit",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_reddit(
|
pub async fn download_reddit(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -211,7 +618,20 @@ pub async fn download_reddit(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _streamable.
|
/// Download Streamable video.
|
||||||
|
/// `/download/streamable?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/streamable",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_streamable",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_streamable(
|
pub async fn download_streamable(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -219,7 +639,20 @@ pub async fn download_streamable(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _videy.
|
/// Download Videy video.
|
||||||
|
/// `/download/videy?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/videy",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_videy",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_videy(
|
pub async fn download_videy(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
@@ -227,10 +660,82 @@ pub async fn download_videy(
|
|||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handler for _bilibili.
|
/// Download Bilibili video.
|
||||||
|
/// `/download/bilibili?url=...`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/download/bilibili",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_download_bilibili",
|
||||||
|
params(DownloadParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Download result", body = DownloadResponse),
|
||||||
|
(status = 400, description = "Bad URL or platform"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn download_bilibili(
|
pub async fn download_bilibili(
|
||||||
Query(params): Query<DownloadParams>,
|
Query(params): Query<DownloadParams>,
|
||||||
) -> Result<Json<DownloadResponse>, AppError> {
|
) -> Result<Json<DownloadResponse>, AppError> {
|
||||||
let result = use_cases::download_bilibili(¶ms.url).await?;
|
let result = use_cases::download_bilibili(¶ms.url).await?;
|
||||||
Ok(Json(DownloadResponse::ok(result)))
|
Ok(Json(DownloadResponse::ok(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ========================================================================
|
||||||
|
// Merged-file static serving
|
||||||
|
// ========================================================================
|
||||||
|
|
||||||
|
/// Serve a merged YouTube MP4 produced by `/download/youtube?merge=true`.
|
||||||
|
/// Path is sanitised: only a plain filename (no `/`, `..`) inside the
|
||||||
|
/// `yt_merge` uploads dir is allowed.
|
||||||
|
///
|
||||||
|
/// `/file/yt_merge/{filename}`
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/file/yt_merge/{filename}",
|
||||||
|
tag = "download",
|
||||||
|
operation_id = "dl_serve_merged_file",
|
||||||
|
params(
|
||||||
|
("filename" = String, Path, description = "Merged MP4 file name")
|
||||||
|
),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Merged MP4 file", content_type = "video/mp4"),
|
||||||
|
(status = 404, description = "File not found or invalid name")
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn serve_merged_file(
|
||||||
|
axum::extract::Path(filename): axum::extract::Path<String>,
|
||||||
|
) -> Result<axum::response::Response, AppError> {
|
||||||
|
// Reject any path traversal or nested segments.
|
||||||
|
if filename.contains('/')
|
||||||
|
|| filename.contains('\\')
|
||||||
|
|| filename.contains("..")
|
||||||
|
|| filename.is_empty()
|
||||||
|
{
|
||||||
|
return Err(AppError::NotFound(format!("invalid file name: {filename}")));
|
||||||
|
}
|
||||||
|
|
||||||
|
let dir = std::path::PathBuf::from("/var/lib/scraper/uploads/yt_merge");
|
||||||
|
let path = dir.join(&filename);
|
||||||
|
let bytes = tokio::fs::read(&path)
|
||||||
|
.await
|
||||||
|
.map_err(|_| AppError::NotFound(format!("file not found: {filename}")))?;
|
||||||
|
|
||||||
|
let content_type = if filename.ends_with(".mp4") {
|
||||||
|
"video/mp4"
|
||||||
|
} else if filename.ends_with(".webm") {
|
||||||
|
"video/webm"
|
||||||
|
} else {
|
||||||
|
"application/octet-stream"
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(axum::response::Response::builder()
|
||||||
|
.header("Content-Type", content_type)
|
||||||
|
.header("Content-Length", bytes.len().to_string())
|
||||||
|
.header(
|
||||||
|
"Content-Disposition",
|
||||||
|
format!("attachment; filename=\"{filename}\""),
|
||||||
|
)
|
||||||
|
.body(axum::body::Body::from(bytes))
|
||||||
|
.map_err(|e| AppError::Internal(format!("build response: {e}")))?)
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
//! Axum handlers for image generation endpoints.
|
||||||
|
//!
|
||||||
|
//! Returns raw image bytes with the proper Content-Type.
|
||||||
|
|
||||||
|
use axum::body::Body;
|
||||||
|
use axum::extract::Query;
|
||||||
|
use axum::http::{header, StatusCode};
|
||||||
|
use axum::response::Response;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use utoipa::IntoParams;
|
||||||
|
|
||||||
|
use crate::application::image as use_cases;
|
||||||
|
use crate::presentation::error::AppError;
|
||||||
|
|
||||||
|
#[derive(Deserialize, IntoParams)]
|
||||||
|
pub struct BratParams {
|
||||||
|
pub text: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /image/brat?text=Hello
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/image/brat",
|
||||||
|
tag = "image",
|
||||||
|
operation_id = "image_brat",
|
||||||
|
params(BratParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Static brat image (PNG)", body = Vec<u8>),
|
||||||
|
(status = 400, description = "Missing text"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn brat_handler(Query(params): Query<BratParams>) -> Result<Response, AppError> {
|
||||||
|
let text = params
|
||||||
|
.text
|
||||||
|
.ok_or_else(|| AppError::BadRequest("text parameter is required".into()))?;
|
||||||
|
let bytes = use_cases::brat(&text).await?;
|
||||||
|
Response::builder()
|
||||||
|
.status(StatusCode::OK)
|
||||||
|
.header(header::CONTENT_TYPE, "image/png")
|
||||||
|
.body(Body::from(bytes))
|
||||||
|
.map_err(|e| AppError::Internal(format!("build response: {e}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /image/brat/animated?text=Hello
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/image/brat/animated",
|
||||||
|
tag = "image",
|
||||||
|
operation_id = "image_brat_animated",
|
||||||
|
params(BratParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Animated brat image (GIF)", body = Vec<u8>),
|
||||||
|
(status = 400, description = "Missing text"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn brat_animated_handler(Query(params): Query<BratParams>) -> Result<Response, AppError> {
|
||||||
|
let text = params
|
||||||
|
.text
|
||||||
|
.ok_or_else(|| AppError::BadRequest("text parameter is required".into()))?;
|
||||||
|
let bytes = use_cases::brat_animated(&text).await?;
|
||||||
|
Response::builder()
|
||||||
|
.status(StatusCode::OK)
|
||||||
|
.header(header::CONTENT_TYPE, "image/gif")
|
||||||
|
.body(Body::from(bytes))
|
||||||
|
.map_err(|e| AppError::Internal(format!("build response: {e}")))
|
||||||
|
}
|
||||||
@@ -7,12 +7,13 @@ use axum::http::StatusCode;
|
|||||||
use axum::Json;
|
use axum::Json;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
use utoipa::IntoParams;
|
||||||
|
|
||||||
use crate::application::misc as use_cases;
|
use crate::application::misc as use_cases;
|
||||||
use crate::presentation::error::AppError;
|
use crate::presentation::error::AppError;
|
||||||
|
|
||||||
/// Query params for currency converter.
|
/// Query params for currency converter.
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize, IntoParams)]
|
||||||
pub struct CurrencyParams {
|
pub struct CurrencyParams {
|
||||||
pub amount: Option<f64>,
|
pub amount: Option<f64>,
|
||||||
pub from: Option<String>,
|
pub from: Option<String>,
|
||||||
@@ -20,6 +21,17 @@ pub struct CurrencyParams {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// GET /misc/currency-converter?amount=100&from=USD&to=IDR
|
/// GET /misc/currency-converter?amount=100&from=USD&to=IDR
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/misc/currency-converter",
|
||||||
|
tag = "misc",
|
||||||
|
operation_id = "currency_converter",
|
||||||
|
params(CurrencyParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Currency conversion result", body = Value),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn currency_converter_handler(
|
pub async fn currency_converter_handler(
|
||||||
Query(params): Query<CurrencyParams>,
|
Query(params): Query<CurrencyParams>,
|
||||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
@@ -32,18 +44,48 @@ pub async fn currency_converter_handler(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// GET /misc/harga-emas
|
/// GET /misc/harga-emas
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/misc/harga-emas",
|
||||||
|
tag = "misc",
|
||||||
|
operation_id = "harga_emas",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Gold prices (Antam)", body = Value),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn harga_emas_handler() -> Result<(StatusCode, Json<Value>), AppError> {
|
pub async fn harga_emas_handler() -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
let result = use_cases::harga_emas().await?;
|
let result = use_cases::harga_emas().await?;
|
||||||
Ok((StatusCode::OK, Json(result)))
|
Ok((StatusCode::OK, Json(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// GET /misc/kurs-bca
|
/// GET /misc/kurs-bca
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/misc/kurs-bca",
|
||||||
|
tag = "misc",
|
||||||
|
operation_id = "kurs_bca",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "BCA exchange rates", body = Value),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn kurs_bca_handler() -> Result<(StatusCode, Json<Value>), AppError> {
|
pub async fn kurs_bca_handler() -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
let result = use_cases::kurs_bca().await?;
|
let result = use_cases::kurs_bca().await?;
|
||||||
Ok((StatusCode::OK, Json(result)))
|
Ok((StatusCode::OK, Json(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// GET /misc/server-info
|
/// GET /misc/server-info
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/misc/server-info",
|
||||||
|
tag = "misc",
|
||||||
|
operation_id = "server_info",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Server info", body = Value),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn server_info_handler() -> Result<(StatusCode, Json<Value>), AppError> {
|
pub async fn server_info_handler() -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
let result = use_cases::server_info().await?;
|
let result = use_cases::server_info().await?;
|
||||||
Ok((StatusCode::OK, Json(result)))
|
Ok((StatusCode::OK, Json(result)))
|
||||||
|
|||||||
@@ -2,8 +2,10 @@ pub mod anime;
|
|||||||
pub mod anime2;
|
pub mod anime2;
|
||||||
pub mod downloader;
|
pub mod downloader;
|
||||||
pub mod health;
|
pub mod health;
|
||||||
|
pub mod image;
|
||||||
pub mod komik;
|
pub mod komik;
|
||||||
pub mod misc;
|
pub mod misc;
|
||||||
pub mod proxy;
|
pub mod proxy;
|
||||||
|
pub mod search;
|
||||||
pub mod stalk;
|
pub mod stalk;
|
||||||
pub mod weebs;
|
pub mod tools;
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
//! Axum handlers for search utilities.
|
||||||
|
//!
|
||||||
|
//! Ported from Shirokami-API `scraper/search/*.js`.
|
||||||
|
|
||||||
|
use axum::extract::Query;
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::Json;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use crate::application::search as use_cases;
|
||||||
|
use crate::presentation::error::AppError;
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct SearchParams {
|
||||||
|
pub query: Option<String>,
|
||||||
|
pub q: Option<String>,
|
||||||
|
pub kota: Option<String>,
|
||||||
|
pub city: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require(v: Option<&String>, name: &str) -> Result<String, AppError> {
|
||||||
|
v.cloned()
|
||||||
|
.ok_or_else(|| AppError::BadRequest(format!("Missing '{}' parameter", name)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /search/bmkg
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/search/bmkg",
|
||||||
|
tag = "search",
|
||||||
|
operation_id = "search_bmkg_handler",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "OK", body = serde_json::Value),
|
||||||
|
(status = 400, description = "Bad Request"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
|
||||||
|
pub async fn bmkg_handler() -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
Ok((StatusCode::OK, Json(use_cases::bmkg().await?)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /search/jadwal-sholat?kota=jakarta
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/search/jadwal-sholat",
|
||||||
|
tag = "search",
|
||||||
|
operation_id = "search_jadwal_sholat_handler",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "OK", body = serde_json::Value),
|
||||||
|
(status = 400, description = "Bad Request"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
|
||||||
|
pub async fn jadwal_sholat_handler(
|
||||||
|
Query(p): Query<SearchParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let kota = require(p.kota.as_ref(), "kota")?;
|
||||||
|
Ok((StatusCode::OK, Json(use_cases::jadwal_sholat(&kota).await?)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /search/weather?city=jakarta
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/search/weather",
|
||||||
|
tag = "search",
|
||||||
|
operation_id = "search_weather_handler",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "OK", body = serde_json::Value),
|
||||||
|
(status = 400, description = "Bad Request"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
|
||||||
|
pub async fn weather_handler(
|
||||||
|
Query(p): Query<SearchParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let city = require(p.city.as_ref().or(p.q.as_ref()), "city")?;
|
||||||
|
Ok((StatusCode::OK, Json(use_cases::weather(&city).await?)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /search/google?query=rust
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/search/google",
|
||||||
|
tag = "search",
|
||||||
|
operation_id = "search_google_handler",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "OK", body = serde_json::Value),
|
||||||
|
(status = 400, description = "Bad Request"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
|
||||||
|
pub async fn google_handler(
|
||||||
|
Query(p): Query<SearchParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let query = require(p.query.as_ref().or(p.q.as_ref()), "query")?;
|
||||||
|
Ok((StatusCode::OK, Json(use_cases::google(&query).await?)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /search/yt?query=rickroll
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/search/yt",
|
||||||
|
tag = "search",
|
||||||
|
operation_id = "search_yt_handler",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "OK", body = serde_json::Value),
|
||||||
|
(status = 400, description = "Bad Request"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
|
||||||
|
pub async fn yt_handler(
|
||||||
|
Query(p): Query<SearchParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let query = require(p.query.as_ref().or(p.q.as_ref()), "query")?;
|
||||||
|
Ok((StatusCode::OK, Json(use_cases::yt_search(&query).await?)))
|
||||||
|
}
|
||||||
@@ -7,12 +7,13 @@ use axum::http::StatusCode;
|
|||||||
use axum::Json;
|
use axum::Json;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
use utoipa::IntoParams;
|
||||||
|
|
||||||
use crate::application::stalk as use_cases;
|
use crate::application::stalk as use_cases;
|
||||||
use crate::presentation::error::AppError;
|
use crate::presentation::error::AppError;
|
||||||
|
|
||||||
/// Query params for stalk endpoints.
|
/// Query params for stalk endpoints.
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize, IntoParams)]
|
||||||
pub struct StalkParams {
|
pub struct StalkParams {
|
||||||
pub username: Option<String>,
|
pub username: Option<String>,
|
||||||
pub user_id: Option<String>,
|
pub user_id: Option<String>,
|
||||||
@@ -22,6 +23,18 @@ pub struct StalkParams {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// GET /stalk/github?username=asepharyana
|
/// GET /stalk/github?username=asepharyana
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/stalk/github",
|
||||||
|
tag = "stalk",
|
||||||
|
operation_id = "stalk_github",
|
||||||
|
params(StalkParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "GitHub user profile", body = Value),
|
||||||
|
(status = 400, description = "Missing username"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn github_handler(
|
pub async fn github_handler(
|
||||||
Query(params): Query<StalkParams>,
|
Query(params): Query<StalkParams>,
|
||||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
@@ -33,6 +46,18 @@ pub async fn github_handler(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// GET /stalk/youtube?username=<handle>
|
/// GET /stalk/youtube?username=<handle>
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/stalk/youtube",
|
||||||
|
tag = "stalk",
|
||||||
|
operation_id = "stalk_youtube",
|
||||||
|
params(StalkParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "YouTube channel profile", body = Value),
|
||||||
|
(status = 400, description = "Missing username"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
pub async fn youtube_handler(
|
pub async fn youtube_handler(
|
||||||
Query(params): Query<StalkParams>,
|
Query(params): Query<StalkParams>,
|
||||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
@@ -42,3 +67,26 @@ pub async fn youtube_handler(
|
|||||||
let result = use_cases::youtube(&username).await?;
|
let result = use_cases::youtube(&username).await?;
|
||||||
Ok((StatusCode::OK, Json(result)))
|
Ok((StatusCode::OK, Json(result)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// GET /stalk/twitter?username=...
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/stalk/twitter",
|
||||||
|
tag = "stalk",
|
||||||
|
operation_id = "stalk_twitter",
|
||||||
|
params(StalkParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Twitter/X user profile", body = Value),
|
||||||
|
(status = 400, description = "Missing username"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn twitter_handler(
|
||||||
|
Query(params): Query<StalkParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let username = params
|
||||||
|
.username
|
||||||
|
.ok_or_else(|| AppError::BadRequest("Missing 'username' parameter".to_string()))?;
|
||||||
|
let result = use_cases::twitter(&username).await?;
|
||||||
|
Ok((StatusCode::OK, Json(result)))
|
||||||
|
}
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
//! Axum handlers for tool endpoints.
|
||||||
|
|
||||||
|
use axum::extract::Query;
|
||||||
|
use axum::http::StatusCode;
|
||||||
|
use axum::Json;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use serde_json::Value;
|
||||||
|
use utoipa::IntoParams;
|
||||||
|
|
||||||
|
use crate::application::tools as use_cases;
|
||||||
|
use crate::presentation::error::AppError;
|
||||||
|
|
||||||
|
#[derive(Deserialize, IntoParams)]
|
||||||
|
pub struct DomainParams {
|
||||||
|
pub domain: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, IntoParams)]
|
||||||
|
pub struct IpParams {
|
||||||
|
pub ip: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, IntoParams)]
|
||||||
|
pub struct UrlParams {
|
||||||
|
pub url: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, IntoParams)]
|
||||||
|
pub struct ResiParams {
|
||||||
|
pub resi: Option<String>,
|
||||||
|
pub ekspedisi: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /tool/whois?domain=example.com
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/tool/whois",
|
||||||
|
tag = "tool",
|
||||||
|
operation_id = "tool_whois",
|
||||||
|
params(DomainParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "WHOIS info", body = Value),
|
||||||
|
(status = 400, description = "Missing domain"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn whois_handler(
|
||||||
|
Query(params): Query<DomainParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let domain = params
|
||||||
|
.domain
|
||||||
|
.ok_or_else(|| AppError::BadRequest("domain parameter is required".into()))?;
|
||||||
|
Ok((StatusCode::OK, Json(use_cases::whois(&domain).await?)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /tool/ip-location?ip=1.1.1.1
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/tool/ip-location",
|
||||||
|
tag = "tool",
|
||||||
|
operation_id = "tool_ip_location",
|
||||||
|
params(IpParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "IP geolocation", body = Value),
|
||||||
|
(status = 400, description = "Missing IP"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn ip_location_handler(
|
||||||
|
Query(params): Query<IpParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let ip = params
|
||||||
|
.ip
|
||||||
|
.ok_or_else(|| AppError::BadRequest("ip parameter is required".into()))?;
|
||||||
|
Ok((StatusCode::OK, Json(use_cases::ip_location(&ip).await?)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /tool/tinyurl?url=https://example.com
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/tool/tinyurl",
|
||||||
|
tag = "tool",
|
||||||
|
operation_id = "tool_tinyurl",
|
||||||
|
params(UrlParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Shortened URL", body = Value),
|
||||||
|
(status = 400, description = "Missing URL"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn tinyurl_handler(
|
||||||
|
Query(params): Query<UrlParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let url = params
|
||||||
|
.url
|
||||||
|
.ok_or_else(|| AppError::BadRequest("url parameter is required".into()))?;
|
||||||
|
Ok((StatusCode::OK, Json(use_cases::tinyurl(&url).await?)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /tool/check-hosting?domain=example.com
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/tool/check-hosting",
|
||||||
|
tag = "tool",
|
||||||
|
operation_id = "tool_check_hosting",
|
||||||
|
params(DomainParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Hosting info", body = Value),
|
||||||
|
(status = 400, description = "Missing domain"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn check_hosting_handler(
|
||||||
|
Query(params): Query<DomainParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let domain = params
|
||||||
|
.domain
|
||||||
|
.ok_or_else(|| AppError::BadRequest("domain parameter is required".into()))?;
|
||||||
|
Ok((
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(use_cases::check_hosting(&domain).await?),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /tool/hargapangan
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/tool/hargapangan",
|
||||||
|
tag = "tool",
|
||||||
|
operation_id = "tool_hargapangan",
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Food prices", body = Value),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn hargapangan_handler() -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
Ok((StatusCode::OK, Json(use_cases::hargapangan().await?)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET /tool/cek-resi?resi=...&ekspedisi=...
|
||||||
|
#[utoipa::path(
|
||||||
|
get,
|
||||||
|
path = "/tool/cek-resi",
|
||||||
|
tag = "tool",
|
||||||
|
operation_id = "tool_cek_resi",
|
||||||
|
params(ResiParams),
|
||||||
|
responses(
|
||||||
|
(status = 200, description = "Shipment tracking", body = Value),
|
||||||
|
(status = 400, description = "Missing resi"),
|
||||||
|
(status = 502, description = "Upstream error"),
|
||||||
|
)
|
||||||
|
)]
|
||||||
|
pub async fn cek_resi_handler(
|
||||||
|
Query(params): Query<ResiParams>,
|
||||||
|
) -> Result<(StatusCode, Json<Value>), AppError> {
|
||||||
|
let ResiParams { resi, ekspedisi } = params;
|
||||||
|
let resi = resi.ok_or_else(|| AppError::BadRequest("resi parameter is required".into()))?;
|
||||||
|
Ok((
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(use_cases::cek_resi(resi, ekspedisi).await?),
|
||||||
|
))
|
||||||
|
}
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
//! Axum handlers for weebs endpoints.
|
|
||||||
//!
|
|
||||||
//! Ported from Shirokami-API `scraper/weebs/*.js`.
|
|
||||||
|
|
||||||
use axum::extract::Query;
|
|
||||||
use axum::http::StatusCode;
|
|
||||||
use axum::Json;
|
|
||||||
use serde::Deserialize;
|
|
||||||
use serde_json::Value;
|
|
||||||
|
|
||||||
use crate::application::weebs as use_cases;
|
|
||||||
use crate::presentation::error::AppError;
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
pub struct WeebsParams {
|
|
||||||
pub query: Option<String>,
|
|
||||||
pub q: Option<String>,
|
|
||||||
pub url: Option<String>,
|
|
||||||
pub tag: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn require<'a>(v: Option<&'a String>, name: &str) -> Result<String, AppError> {
|
|
||||||
v.cloned()
|
|
||||||
.ok_or_else(|| AppError::BadRequest(format!("Missing '{}' parameter", name)))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// GET /weebs/anime-info?query=naruto
|
|
||||||
pub async fn anime_info_handler(
|
|
||||||
Query(p): Query<WeebsParams>,
|
|
||||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
|
||||||
let query = require(p.query.as_ref().or(p.q.as_ref()), "query")?;
|
|
||||||
Ok((StatusCode::OK, Json(use_cases::anime_info(&query).await?)))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// GET /weebs/manga-info?query=one%20piece
|
|
||||||
pub async fn manga_info_handler(
|
|
||||||
Query(p): Query<WeebsParams>,
|
|
||||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
|
||||||
let query = require(p.query.as_ref().or(p.q.as_ref()), "query")?;
|
|
||||||
Ok((StatusCode::OK, Json(use_cases::manga_info(&query).await?)))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// GET /weebs/whatanime?url=<image-url>
|
|
||||||
pub async fn whatanime_handler(
|
|
||||||
Query(p): Query<WeebsParams>,
|
|
||||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
|
||||||
let url = require(p.url.as_ref(), "url")?;
|
|
||||||
Ok((StatusCode::OK, Json(use_cases::whatanime(&url).await?)))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// GET /weebs/sfw-waifu?tag=waifu
|
|
||||||
pub async fn sfw_waifu_handler(
|
|
||||||
Query(p): Query<WeebsParams>,
|
|
||||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
|
||||||
let tag = require(p.tag.as_ref(), "tag")?;
|
|
||||||
Ok((StatusCode::OK, Json(use_cases::sfw_waifu(&tag).await?)))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// GET /weebs/nsfw-waifu?tag=waifu
|
|
||||||
pub async fn nsfw_waifu_handler(
|
|
||||||
Query(p): Query<WeebsParams>,
|
|
||||||
) -> Result<(StatusCode, Json<Value>), AppError> {
|
|
||||||
let tag = require(p.tag.as_ref(), "tag")?;
|
|
||||||
Ok((StatusCode::OK, Json(use_cases::nsfw_waifu(&tag).await?)))
|
|
||||||
}
|
|
||||||
+57
-11
@@ -184,6 +184,10 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
|||||||
"/download/youtube/mp3",
|
"/download/youtube/mp3",
|
||||||
axum::routing::get(crate::presentation::handler::downloader::download_youtube_mp3),
|
axum::routing::get(crate::presentation::handler::downloader::download_youtube_mp3),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/file/yt_merge/{filename}",
|
||||||
|
axum::routing::get(crate::presentation::handler::downloader::serve_merged_file),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/download/spotify",
|
"/download/spotify",
|
||||||
axum::routing::get(crate::presentation::handler::downloader::download_spotify),
|
axum::routing::get(crate::presentation::handler::downloader::download_spotify),
|
||||||
@@ -204,6 +208,10 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
|||||||
"/download/terabox",
|
"/download/terabox",
|
||||||
axum::routing::get(crate::presentation::handler::downloader::download_terabox),
|
axum::routing::get(crate::presentation::handler::downloader::download_terabox),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/proxy/terabox",
|
||||||
|
axum::routing::get(crate::presentation::handler::downloader::proxy_terabox),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/download/gdrive",
|
"/download/gdrive",
|
||||||
axum::routing::get(crate::presentation::handler::downloader::download_gdrive),
|
axum::routing::get(crate::presentation::handler::downloader::download_gdrive),
|
||||||
@@ -282,26 +290,64 @@ pub fn build_router(app_state: Arc<AppState>) -> anyhow::Result<Router> {
|
|||||||
"/stalk/youtube",
|
"/stalk/youtube",
|
||||||
axum::routing::get(crate::presentation::handler::stalk::youtube_handler),
|
axum::routing::get(crate::presentation::handler::stalk::youtube_handler),
|
||||||
)
|
)
|
||||||
// Weebs routes
|
|
||||||
.route(
|
.route(
|
||||||
"/weebs/anime-info",
|
"/stalk/twitter",
|
||||||
axum::routing::get(crate::presentation::handler::weebs::anime_info_handler),
|
axum::routing::get(crate::presentation::handler::stalk::twitter_handler),
|
||||||
|
)
|
||||||
|
// Search routes
|
||||||
|
.route(
|
||||||
|
"/search/bmkg",
|
||||||
|
axum::routing::get(crate::presentation::handler::search::bmkg_handler),
|
||||||
)
|
)
|
||||||
.route(
|
.route(
|
||||||
"/weebs/manga-info",
|
"/search/jadwal-sholat",
|
||||||
axum::routing::get(crate::presentation::handler::weebs::manga_info_handler),
|
axum::routing::get(crate::presentation::handler::search::jadwal_sholat_handler),
|
||||||
)
|
)
|
||||||
.route(
|
.route(
|
||||||
"/weebs/whatanime",
|
"/search/weather",
|
||||||
axum::routing::get(crate::presentation::handler::weebs::whatanime_handler),
|
axum::routing::get(crate::presentation::handler::search::weather_handler),
|
||||||
)
|
)
|
||||||
.route(
|
.route(
|
||||||
"/weebs/sfw-waifu",
|
"/search/google",
|
||||||
axum::routing::get(crate::presentation::handler::weebs::sfw_waifu_handler),
|
axum::routing::get(crate::presentation::handler::search::google_handler),
|
||||||
)
|
)
|
||||||
.route(
|
.route(
|
||||||
"/weebs/nsfw-waifu",
|
"/search/yt",
|
||||||
axum::routing::get(crate::presentation::handler::weebs::nsfw_waifu_handler),
|
axum::routing::get(crate::presentation::handler::search::yt_handler),
|
||||||
|
)
|
||||||
|
// Tool routes
|
||||||
|
.route(
|
||||||
|
"/tool/whois",
|
||||||
|
axum::routing::get(crate::presentation::handler::tools::whois_handler),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/tool/iplocation",
|
||||||
|
axum::routing::get(crate::presentation::handler::tools::ip_location_handler),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/tool/tinyurl",
|
||||||
|
axum::routing::get(crate::presentation::handler::tools::tinyurl_handler),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/tool/check-hosting",
|
||||||
|
axum::routing::get(crate::presentation::handler::tools::check_hosting_handler),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/tool/hargapangan",
|
||||||
|
axum::routing::get(crate::presentation::handler::tools::hargapangan_handler),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/tool/cek-resi",
|
||||||
|
axum::routing::get(crate::presentation::handler::tools::cek_resi_handler),
|
||||||
|
)
|
||||||
|
// Image routes
|
||||||
|
.route(
|
||||||
|
"/image/brat",
|
||||||
|
axum::routing::get(crate::presentation::handler::image::brat_handler),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/image/brat/animated",
|
||||||
|
axum::routing::get(crate::presentation::handler::image::brat_animated_handler),
|
||||||
)
|
)
|
||||||
// Health
|
// Health
|
||||||
.route(
|
.route(
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""Extract TikTok user-detail JSON (__UNIVERSAL_DATA_FOR_REHYDRATION__) via Playwright.
|
||||||
|
|
||||||
|
The direct HTTP path gets a SlardarWAF challenge page; a real browser with JS
|
||||||
|
executes the challenge and exposes the __UNIVERSAL_DATA_FOR_REHYDRATION__ script.
|
||||||
|
Usage: python3 stalk_tiktok.py <username>
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.environ.setdefault("PLAYWRIGHT_BROWSERS_PATH", "/usr/local/share/ms-playwright")
|
||||||
|
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
|
|
||||||
|
async def main(username):
|
||||||
|
url = f"https://www.tiktok.com/@{username}?_t=ZS-8tHANz7ieoS&_r=1"
|
||||||
|
async with async_playwright() as p:
|
||||||
|
browser = await p.chromium.launch(
|
||||||
|
headless=True,
|
||||||
|
args=["--no-sandbox", "--disable-bypass", "--disable-blink-features=AutomationControlled"],
|
||||||
|
)
|
||||||
|
context = await browser.new_context(
|
||||||
|
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
viewport={"width": 1366, "height": 768},
|
||||||
|
java_script_enabled=True,
|
||||||
|
locale="en-US",
|
||||||
|
)
|
||||||
|
page = await context.new_page()
|
||||||
|
try:
|
||||||
|
await page.goto(url, timeout=60000, wait_until="domcontentloaded")
|
||||||
|
except Exception as e:
|
||||||
|
await browser.close()
|
||||||
|
print(json.dumps({"success": False, "error": f"Navigation: {e}"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
# Wait for the rehydration script to appear (WAF challenge may take a few seconds)
|
||||||
|
found = None
|
||||||
|
for _ in range(30):
|
||||||
|
try:
|
||||||
|
found = await page.evaluate(
|
||||||
|
"""() => {
|
||||||
|
const s = document.querySelector('script#__UNIVERSAL_DATA_FOR_REHYDRATION__');
|
||||||
|
return s ? s.textContent : null;
|
||||||
|
}"""
|
||||||
|
)
|
||||||
|
if found:
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
await page.wait_for_timeout(1000)
|
||||||
|
|
||||||
|
await browser.close()
|
||||||
|
|
||||||
|
if not found:
|
||||||
|
print(json.dumps({"success": False, "error": "Rehydration data not found (WAF challenge)"}))
|
||||||
|
return
|
||||||
|
|
||||||
|
print(json.dumps({"success": True, "data": found}))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main(sys.argv[1]))
|
||||||
Reference in new issue
Block a user