From 242e3498fe8c82fec8971312210565f8cabb1b4a Mon Sep 17 00:00:00 2001 From: asepharyana Date: Tue, 1 Sep 2026 19:31:37 +0700 Subject: [PATCH] feat(stalk): add github + youtube profile stalk endpoints Ported from Shirokami-API scraper/stalk/{github,youtube}.js: - /stalk/github (GitHub public API profile) - /stalk/youtube (ytInitialData parse from /videos subpage: handles lockupViewModel (new), videoRenderer, gridVideoRenderer, shortsLockupViewModel; channel metadata + top-5 videos) Also fix misc harga-emas main price to smallest-gram (1gr). --- src/application/mod.rs | 1 + src/application/stalk.rs | 19 ++ src/infrastructure/repository/mod.rs | 1 + src/infrastructure/repository/stalk.rs | 403 +++++++++++++++++++++++++ src/presentation/handler/mod.rs | 1 + src/presentation/handler/stalk.rs | 44 +++ src/presentation/router.rs | 9 + 7 files changed, 478 insertions(+) create mode 100644 src/application/stalk.rs create mode 100644 src/infrastructure/repository/stalk.rs create mode 100644 src/presentation/handler/stalk.rs diff --git a/src/application/mod.rs b/src/application/mod.rs index 3529315..f4ca339 100644 --- a/src/application/mod.rs +++ b/src/application/mod.rs @@ -4,3 +4,4 @@ pub mod downloader; pub mod komik; pub mod misc; pub mod proxy; +pub mod stalk; diff --git a/src/application/stalk.rs b/src/application/stalk.rs new file mode 100644 index 0000000..2965205 --- /dev/null +++ b/src/application/stalk.rs @@ -0,0 +1,19 @@ +//! Application use-cases for stalk (profile/user lookups). + +use crate::domain::error::ScrapingError; +use crate::infrastructure::repository::stalk as Repo; +use serde_json::Value; + +/// GitHub user profile stalk. +pub async fn github(username: &str) -> Result { + Repo::fetch_github_stalk(username) + .await + .map_err(ScrapingError::Http) +} + +/// YouTube channel/profile stalk. +pub async fn youtube(username: &str) -> Result { + Repo::fetch_youtube_stalk(username) + .await + .map_err(ScrapingError::Http) +} diff --git a/src/infrastructure/repository/mod.rs b/src/infrastructure/repository/mod.rs index 1ec2359..fd83ab2 100644 --- a/src/infrastructure/repository/mod.rs +++ b/src/infrastructure/repository/mod.rs @@ -5,6 +5,7 @@ pub mod misc; pub mod otakudesu; pub mod parsers; pub mod proxy; +pub mod stalk; pub use alqanime::AlqanimeRepository; pub use downloader::DownloaderRepository; diff --git a/src/infrastructure/repository/stalk.rs b/src/infrastructure/repository/stalk.rs new file mode 100644 index 0000000..97ea4c5 --- /dev/null +++ b/src/infrastructure/repository/stalk.rs @@ -0,0 +1,403 @@ +//! Infrastructure — Stalk utilities (github, youtube, instagram, tiktok, twitter). +//! +//! Ported from Shirokami-API `scraper/stalk/*.js`. + +use crate::infrastructure::utils::http_client::http_client; +use regex::Regex; +use reqwest::header::USER_AGENT; +use serde_json::{json, Value}; + +/// GitHub user stalk via public API. +pub async fn fetch_github_stalk(username: &str) -> Result { + let url = format!("https://api.github.com/users/{}", username); + + let resp = http_client() + .client() + .get(&url) + .header(USER_AGENT, "Scraper/1.0") + .send() + .await + .map_err(|e| format!("HTTP error: {}", e))?; + + if !resp.status().is_success() { + let status = resp.status().as_u16(); + return Err(format!("GitHub user not found (HTTP {})", status)); + } + + let data: Value = resp + .json() + .await + .map_err(|e| format!("JSON parse: {}", e))?; + + let pick = |k: &str| -> Value { data.get(k).cloned().unwrap_or(Value::Null) }; + + Ok(json!({ + "username": username, + "login": pick("login"), + "id": pick("id"), + "node_id": pick("node_id"), + "avatar_url": pick("avatar_url"), + "html_url": pick("html_url"), + "type": pick("type"), + "site_admin": pick("site_admin"), + "name": pick("name"), + "company": pick("company"), + "blog": pick("blog"), + "location": pick("location"), + "email": pick("email"), + "hireable": pick("hireable"), + "bio": pick("bio"), + "twitter_username": pick("twitter_username"), + "public_repos": pick("public_repos"), + "public_gists": pick("public_gists"), + "followers": pick("followers"), + "following": pick("following"), + "created_at": pick("created_at"), + "updated_at": pick("updated_at"), + })) +} + +/// YouTube channel stalk: parse ytInitialData JSON from channel page. +/// Fetches the `//videos` subpage (richGridRenderer with lockupViewModel). +pub async fn fetch_youtube_stalk(username: &str) -> Result { + let url = format!("https://youtube.com/@{username}/videos?hl=en&gl=US"); + + let resp = http_client() + .client() + .get(&url) + .header(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36") + .header("Accept-Language", "en-US") + .send() + .await + .map_err(|e| format!("HTTP error: {}", e))?; + + if !resp.status().is_success() { + return Err(format!("HTTP {} from youtube", resp.status())); + } + + let body = resp + .text() + .await + .map_err(|e| format!("Failed to read body: {}", e))?; + + // Extract var ytInitialData = {...} via brace matching (JSON may contain ";"). + let start = body.find("var ytInitialData = ").ok_or_else(|| "ytInitialData not found".to_string())?; + let brace = body[start..].find('{').ok_or_else(|| "ytInitialData parse".to_string())? + start; + let json_str = extract_balanced_json(&body, brace)?; + let parsed: Value = + serde_json::from_str(&json_str).map_err(|e| format!("JSON parse: {}", e))?; + + // Channel metadata from metadata.channelMetadataRenderer + let mut channel = json!({ + "username": username, + "subscriberCount": Value::Null, + "videoCount": Value::Null, + "avatarUrl": Value::Null, + "channelUrl": Value::Null, + "externalId": Value::Null, + "description": Value::Null, + "isFamilySafe": Value::Null, + }); + + if let Some(meta) = parsed.get("metadata").and_then(|m| m.get("channelMetadataRenderer")) { + let mut set = |dst: &str, src: &str| { + if let Some(v) = meta.get(src) { + channel[dst] = v.clone(); + } + }; + set("description", "description"); + set("externalId", "externalId"); + set("channelUrl", "channelUrl"); + set("isFamilySafe", "isFamilySafe"); + } + + // Subscriber/video count from header.pageHeaderRenderer + if let Some(header) = parsed.get("header").and_then(|h| h.get("pageHeaderRenderer")) { + if let Some(rows) = header + .pointer("/content/pageHeaderViewModel/metadata/contentMetadataViewModel/metadataRows") + { + if let Some(arr) = rows.as_array() { + for row in arr { + if let Some(parts) = row.get("metadataParts").and_then(|p| p.as_array()) { + for part in parts { + let text = part + .pointer("/text/content") + .and_then(|t| t.as_str()) + .unwrap_or(""); + if text.contains("subscriber") { + channel["subscriberCount"] = json!(text.trim()); + } else if text.contains(" video") || text.ends_with(" videos") { + channel["videoCount"] = json!(text.trim()); + } + } + } + } + } + } + // Avatar + if let Some(sources) = header + .pointer("/content/pageHeaderViewModel/image/decoratedAvatarViewModel/avatar/avatarViewModel/image/sources") + .and_then(|s| s.as_array()) + { + if !sources.is_empty() { + channel["avatarUrl"] = sources[0].get("url").cloned().unwrap_or(Value::Null); + } + } + } + + // Videos list (up to 5). The `/videos` subpage uses richGridRenderer with + // lockupViewModel (new) or videoRenderer (legacy); we handle both. + let mut videos: Vec = Vec::new(); + + if let Some(tabs) = parsed + .pointer("/contents/twoColumnBrowseResultsRenderer/tabs") + .and_then(|t| t.as_array()) + { + for tab in tabs { + walk_video_grid(tab, &mut videos); + if videos.len() >= 5 { + break; + } + } + } + + // Fallback: source's shelfRenderer/horizontalListRenderer approach. + if videos.is_empty() { + if let Some(tabs) = parsed + .pointer("/contents/twoColumnBrowseResultsRenderer/tabs") + .and_then(|t| t.as_array()) + { + if let Some(contents) = tabs + .get(0) + .and_then(|t| t.pointer("/tabRenderer/content/sectionListRenderer/contents")) + .and_then(|c| c.as_array()) + { + for item in contents { + if let Some(section) = item.get("itemSectionRenderer") { + if let Some(inner) = section.get("contents").and_then(|c| c.as_array()) { + for content in inner { + if let Some(shelf) = content.get("shelfRenderer") { + if let Some(items) = shelf + .pointer("/content/horizontalListRenderer/items") + .and_then(|i| i.as_array()) + { + for video in items { + if let Some(gv) = video.get("gridVideoRenderer") { + videos.push(video_from_grid_video(gv)); + } + } + } + } + } + } + } + if videos.len() >= 5 { + break; + } + } + } + } + } + + Ok(json!({ + "channelMetadata": channel, + "videoDataList": videos.iter().take(5).cloned().collect::>(), + })) +} + +/// Extract a balanced JSON object string starting at `open_brace` (index of '{'). +fn extract_balanced_json(body: &str, open_brace: usize) -> Result { + let bytes = body.as_bytes(); + let mut depth = 0i32; + let mut in_str = false; + let mut escaped = false; + let mut close = None; + for i in open_brace..bytes.len() { + let c = bytes[i] as char; + if in_str { + if escaped { + escaped = false; + } else if c == '\\' { + escaped = true; + } else if c == '"' { + in_str = false; + } + } else { + match c { + '"' => in_str = true, + '{' => depth += 1, + '}' => { + depth -= 1; + if depth == 0 { + close = Some(i); + break; + } + } + _ => {} + } + } + } + let close = close.ok_or_else(|| "unbalanced JSON in ytInitialData".to_string())?; + Ok(body[open_brace..=close].to_string()) +} + +/// Recursively find `richItemRenderer` entries and convert their content to video objects. +fn walk_video_grid(node: &Value, out: &mut Vec) { + match node { + Value::Object(map) => { + if let Some(item) = map.get("richItemRenderer") { + if let Some(content) = item.get("content") { + if let Some(v) = content.get("videoRenderer") { + out.push(video_from_video_renderer(v)); + } else if let Some(lv) = content.get("lockupViewModel") { + out.push(video_from_lockup(lv)); + } else if let Some(sh) = content.get("shortsLockupViewModel") { + out.push(video_from_shorts(sh)); + } + } + } + for v in map.values() { + if out.len() >= 5 { + return; + } + walk_video_grid(v, out); + } + } + Value::Array(arr) => { + for v in arr { + if out.len() >= 5 { + return; + } + walk_video_grid(v, out); + } + } + _ => {} + } +} + +/// Parse a legacy `videoRenderer` into our video shape. +fn video_from_video_renderer(vd: &Value) -> Value { + let mut v = json!({ + "videoId": vd.get("videoId").cloned().unwrap_or(Value::Null), + "title": vd.pointer("/title/runs/0/text").cloned().unwrap_or(Value::Null), + "thumbnail": vd.pointer("/thumbnail/thumbnails/0/url").cloned().unwrap_or(Value::Null), + "publishedTime": vd.pointer("/publishedTimeText/simpleText").cloned().unwrap_or(Value::Null), + "viewCount": vd.pointer("/viewCountText/simpleText").cloned().unwrap_or(Value::Null), + "navigationUrl": Value::Null, + "duration": Value::Null, + }); + if let Some(url) = vd.pointer("/navigationEndpoint/commandMetadata/webCommandMetadata/url") { + v["navigationUrl"] = url.clone(); + } + if let Some(dur) = vd + .pointer("/thumbnailOverlays") + .and_then(|o| o.as_array()) + .and_then(|arr| { + arr.iter() + .find_map(|ov| ov.pointer("/thumbnailOverlayTimeStatusRenderer/text/simpleText").cloned()) + }) + { + v["duration"] = dur; + } + v +} + +/// Parse a `lockupViewModel` (newer YouTube renderer) into our video shape. +fn video_from_lockup(lv: &Value) -> Value { + // videoId from thumbnail URL `i.ytimg.com/vi//...` + let thumb = lv + .pointer("/contentImage/thumbnailViewModel/image/sources/0/url") + .and_then(|u| u.as_str()) + .unwrap_or(""); + let video_id = Regex::new(r"vi/([A-Za-z0-9_-]{11})/") + .ok() + .and_then(|re| re.captures(thumb)) + .and_then(|c| c.get(1)) + .map(|m| m.as_str().to_string()) + .unwrap_or_default(); + + let title = lv + .pointer("/metadata/lockupMetadataViewModel/title/content") + .and_then(|t| t.as_str()) + .unwrap_or("") + .to_string(); + + let mut view_count = Value::Null; + let mut published = Value::Null; + if let Some(parts) = lv + .pointer("/metadata/lockupMetadataViewModel/metadata/contentMetadataViewModel/metadataRows/0/metadataParts") + .and_then(|p| p.as_array()) + { + if let Some(p0) = parts.get(0) { + view_count = p0 + .pointer("/text/content") + .cloned() + .unwrap_or(Value::Null); + } + if let Some(p1) = parts.get(1) { + published = p1 + .pointer("/text/content") + .cloned() + .unwrap_or(Value::Null); + } + } + + let duration = lv + .pointer("/contentImage/thumbnailViewModel/image/overlays/0/thumbnailBottomOverlayViewModel/badges/0/thumbnailBadgeViewModel/text") + .cloned() + .unwrap_or(Value::Null); + + json!({ + "videoId": json!(video_id), + "title": json!(title), + "thumbnail": json!(thumb), + "publishedTime": published, + "viewCount": view_count, + "navigationUrl": Value::Null, + "duration": duration, + }) +} + +/// Parse a `shortsLockupViewModel` (Shorts) into our video shape (best-effort). +fn video_from_shorts(sh: &Value) -> Value { + let video_id = sh + .pointer("/overlayMetadata/primaryText/content") + .and_then(|c| c.as_str()) + .unwrap_or(""); + json!({ + "videoId": json!(video_id), + "title": Value::Null, + "thumbnail": Value::Null, + "publishedTime": Value::Null, + "viewCount": Value::Null, + "navigationUrl": Value::Null, + "duration": Value::Null, + }) +} + +/// Parse a legacy `gridVideoRenderer` into our video shape. +fn video_from_grid_video(gv: &Value) -> Value { + let mut v = json!({ + "videoId": gv.get("videoId").cloned().unwrap_or(Value::Null), + "title": gv.pointer("/title/simpleText").cloned().unwrap_or(Value::Null), + "thumbnail": gv.pointer("/thumbnail/thumbnails/0/url").cloned().unwrap_or(Value::Null), + "publishedTime": gv.pointer("/publishedTimeText/simpleText").cloned().unwrap_or(Value::Null), + "viewCount": gv.pointer("/viewCountText/simpleText").cloned().unwrap_or(Value::Null), + "navigationUrl": Value::Null, + "duration": Value::Null, + }); + if let Some(nav) = gv.pointer("/navigationEndpoint/commandMetadata/webCommandMetadata/url") { + v["navigationUrl"] = nav.clone(); + } + if let Some(dur) = gv + .pointer("/thumbnailOverlays") + .and_then(|o| o.as_array()) + .and_then(|arr| { + arr.iter() + .find_map(|ov| ov.pointer("/thumbnailOverlayTimeStatusRenderer/text/simpleText").cloned()) + }) + { + v["duration"] = dur; + } + v +} diff --git a/src/presentation/handler/mod.rs b/src/presentation/handler/mod.rs index 4429351..cdeac2a 100644 --- a/src/presentation/handler/mod.rs +++ b/src/presentation/handler/mod.rs @@ -5,3 +5,4 @@ pub mod health; pub mod komik; pub mod misc; pub mod proxy; +pub mod stalk; diff --git a/src/presentation/handler/stalk.rs b/src/presentation/handler/stalk.rs new file mode 100644 index 0000000..35363fb --- /dev/null +++ b/src/presentation/handler/stalk.rs @@ -0,0 +1,44 @@ +//! Axum handlers for stalk (profile/user lookups). +//! +//! Ported from Shirokami-API `scraper/stalk/*.js`. + +use axum::extract::Query; +use axum::http::StatusCode; +use axum::Json; +use serde::Deserialize; +use serde_json::Value; + +use crate::application::stalk as use_cases; +use crate::presentation::error::AppError; + +/// Query params for stalk endpoints. +#[derive(Deserialize)] +pub struct StalkParams { + pub username: Option, + pub user_id: Option, + pub userid: Option, + pub zone_id: Option, + pub zoneid: Option, +} + +/// GET /stalk/github?username=asepharyana +pub async fn github_handler( + Query(params): Query, +) -> Result<(StatusCode, Json), AppError> { + let username = params + .username + .ok_or_else(|| AppError::BadRequest("Missing 'username' parameter".to_string()))?; + let result = use_cases::github(&username).await?; + Ok((StatusCode::OK, Json(result))) +} + +/// GET /stalk/youtube?username= +pub async fn youtube_handler( + Query(params): Query, +) -> Result<(StatusCode, Json), AppError> { + let username = params + .username + .ok_or_else(|| AppError::BadRequest("Missing 'username' parameter".to_string()))?; + let result = use_cases::youtube(&username).await?; + Ok((StatusCode::OK, Json(result))) +} diff --git a/src/presentation/router.rs b/src/presentation/router.rs index 61bf767..86b9745 100644 --- a/src/presentation/router.rs +++ b/src/presentation/router.rs @@ -273,6 +273,15 @@ pub fn build_router(app_state: Arc) -> anyhow::Result { "/misc/server-info", axum::routing::get(crate::presentation::handler::misc::server_info_handler), ) + // Stalk routes + .route( + "/stalk/github", + axum::routing::get(crate::presentation::handler::stalk::github_handler), + ) + .route( + "/stalk/youtube", + axum::routing::get(crate::presentation::handler::stalk::youtube_handler), + ) // Health .route( "/health",