MythEclipse/scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remove croxy proxy and image cache features

Browse Source
This commit is contained in:
asepharyana
2026-07-22 14:51:08 +07:00
parent 891064354b
commit 5c51ec895f
35 changed files with 42 additions and 2799 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/application/anime/use_cases.rs
+3 -72
View File
@@ -3,18 +3,12 @@
//! Orchestrates repository fetching, caching, and image poster processing.
//! Returns pure domain types — no DTOs.
use std::sync::Arc;
use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::entity::anime::*;
use crate::domain::error::*;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::OtakudesuRepository;
use crate::infrastructure::services::images::cache::{
cache_image_urls_batch_lazy, get_cached_or_original,
};
const INDEX_CACHE_TTL: u64 = 10;
const GENRE_LIST_CACHE_TTL: u64 = 3600;
@@ -23,22 +17,13 @@ const DEFAULT_CACHE_TTL: u64 = 300;
pub struct AnimeUseCases {
repository: OtakudesuRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
}
impl AnimeUseCases {
pub fn new(
repository: OtakudesuRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
pub fn new(repository: OtakudesuRepository, redis_pool: Pool) -> Self {
Self {
repository,
redis_pool,
db,
semaphore,
}
}
@@ -49,7 +34,7 @@ impl AnimeUseCases {
pub async fn get_anime_index(&self) -> Result<AnimeData, DomainError> {
self.cache()
.get_or_set("anime:index:v2", INDEX_CACHE_TTL, || async {
let mut data = self
let data = self
.repository
.fetch_anime_index()
.await
@@ -59,33 +44,6 @@ impl AnimeUseCases {
return Err("Empty anime index — refusing to cache".to_string());
}
let mut posters: Vec<String> = data
.ongoing_anime
.iter()
.map(|item| item.poster.clone())
.collect();
posters.extend(data.complete_anime.iter().map(|item| item.poster.clone()));
let cached_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
posters,
self.semaphore.clone(),
)
.await;
let ongoing_len = data.ongoing_anime.len();
for (i, item) in data.ongoing_anime.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(i) {
item.poster = url.clone();
}
}
for (i, item) in data.complete_anime.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(ongoing_len + i) {
item.poster = url.clone();
}
}
Ok(data)
})
.await
@@ -108,39 +66,12 @@ impl AnimeUseCases {
let cache_key = format!("anime:detail:{}", slug);
self.cache()
.get_or_set(&cache_key, DEFAULT_CACHE_TTL, || async {
let mut data = self
let data = self
.repository
.fetch_anime_detail(&slug)
.await
.map_err(|e| e.to_string())?;
data.poster = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster,
self.semaphore.clone(),
)
.await;
let rec_posters: Vec<String> = data
.recommendations
.iter()
.map(|r| r.poster.clone())
.collect();
let cached_rec_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
rec_posters,
self.semaphore.clone(),
)
.await;
for (i, rec) in data.recommendations.iter_mut().enumerate() {
if let Some(url) = cached_rec_posters.get(i) {
rec.poster = url.clone();
}
}
Ok(data)
})
.await
src/application/anime2/use_cases.rs
+10 -129
View File
@@ -8,23 +8,17 @@
//! TODO: Once parsers return domain types, replace shared types with
//! `crate::domain::entity::anime::{GenreAnimeItem, SearchAnimeItem, LatestAnimeItem}`.
use std::sync::Arc;
use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::error::*;
use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::AlqanimeRepository;
use crate::infrastructure::services::images::cache::{
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
};
use crate::infrastructure::repository::parsers::alqanime_parser as parser;
use crate::domain::entity::anime::{
CompleteAnimeItem, FilterAnimeItem, Genre, GenreAnimeItem, HasPoster, LatestAnimeItem,
CompleteAnimeItem, FilterAnimeItem, Genre, GenreAnimeItem, LatestAnimeItem,
OngoingAnimeItemWithScore, Pagination, SearchAnimeItem,
};
@@ -56,15 +50,6 @@ pub struct Anime2Item {
pub anime_url: String,
}
impl HasPoster for Anime2Item {
fn poster(&self) -> &str {
&self.poster
}
fn set_poster(&mut self, url: String) {
self.poster = url;
}
}
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Anime2Data {
pub ongoing_anime: Vec<Anime2Item>,
@@ -123,22 +108,13 @@ const COMPLETE_CACHE_TTL: u64 = 300;
pub struct Anime2UseCases {
repository: AlqanimeRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
}
impl Anime2UseCases {
pub fn new(
repository: AlqanimeRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
pub fn new(repository: AlqanimeRepository, redis_pool: Pool) -> Self {
Self {
repository,
redis_pool,
db,
semaphore,
}
}
@@ -168,7 +144,7 @@ impl Anime2UseCases {
})
.await
.map_err(|e| e.to_string())??;
let mut ongoing: Vec<Anime2Item> = data
let ongoing: Vec<Anime2Item> = data
.0
.into_iter()
.map(|item| Anime2Item {
@@ -182,7 +158,7 @@ impl Anime2UseCases {
})
.collect();
let mut complete: Vec<Anime2Item> = data
let complete: Vec<Anime2Item> = data
.1
.into_iter()
.map(|item| Anime2Item {
@@ -196,30 +172,6 @@ impl Anime2UseCases {
})
.collect();
let mut posters: Vec<String> =
ongoing.iter().map(|item| item.poster.clone()).collect();
posters.extend(complete.iter().map(|item| item.poster.clone()));
let cached_posters = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
posters,
self.semaphore.clone(),
)
.await;
let ongoing_len = ongoing.len();
for (i, item) in ongoing.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(i) {
item.poster = url.clone();
}
}
for (i, item) in complete.iter_mut().enumerate() {
if let Some(url) = cached_posters.get(ongoing_len + i) {
item.poster = url.clone();
}
}
Ok(Anime2Response {
status: "Ok".to_string(),
data: Anime2Data {
@@ -293,20 +245,12 @@ impl Anime2UseCases {
.fetch_html(&url)
.await
.map_err(|e| e.to_string())?;
let (mut data, pagination) = tokio::task::spawn_blocking(move || {
let (data, pagination) = tokio::task::spawn_blocking(move || {
parser::parse_filter_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(FilterResponse {
success: true,
data,
@@ -374,29 +318,6 @@ impl Anime2UseCases {
}
}
data.poster = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster,
self.semaphore.clone(),
)
.await;
data.poster2 = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster2,
self.semaphore.clone(),
)
.await;
apply_cached_posters(
&mut data.recommendations,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(DetailResponse {
status: "Ok".to_string(),
data,
@@ -420,20 +341,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.genre_page_url(&genre_slug, page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_genre_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
@@ -454,20 +367,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.search_url(&query, page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_search_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
@@ -487,20 +392,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.latest_url(page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_latest_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
@@ -520,20 +417,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.ongoing_url(page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_ongoing_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
@@ -553,20 +442,12 @@ impl Anime2UseCases {
.fetch_html(&self.repository.complete_url(page))
.await
.map_err(|e| e.to_string())?;
let (mut data, _pagination) = tokio::task::spawn_blocking(move || {
let (data, _pagination) = tokio::task::spawn_blocking(move || {
parser::parse_complete_page(&html, page).map_err(|e| e.to_string())
})
.await
.map_err(|e| e.to_string())??;
apply_cached_posters(
&mut data,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok(ApiResponse::success(data))
})
.await
src/application/komik/use_cases.rs
+8 -81
View File
@@ -6,10 +6,7 @@
//! `crate::infrastructure::repository::parsers::komik_parser`.
//! TODO: Move response DTOs to `crate::presentation::dto::komik`.
use std::sync::Arc;
use deadpool_redis::Pool;
use sea_orm::DatabaseConnection;
use crate::domain::entity::anime::Pagination;
use crate::domain::entity::komik::{ChapterData, DetailData, KomikGenre, KomikItem};
@@ -17,9 +14,6 @@ use crate::domain::error::*;
use crate::domain::repository::ScrapingRepository;
use crate::infrastructure::cache::redis::Cache;
use crate::infrastructure::repository::KomikRepository;
use crate::infrastructure::services::images::cache::{
apply_cached_posters, cache_image_urls_batch_lazy, get_cached_or_original,
};
use crate::infrastructure::repository::parsers::komik_parser as parser;
@@ -36,22 +30,13 @@ const SEARCH_CACHE_TTL: u64 = 300;
pub struct KomikUseCases {
repository: KomikRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
}
impl KomikUseCases {
pub fn new(
repository: KomikRepository,
redis_pool: Pool,
db: Arc<DatabaseConnection>,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Self {
pub fn new(repository: KomikRepository, redis_pool: Pool) -> Self {
Self {
repository,
redis_pool,
db,
semaphore,
}
}
@@ -94,20 +79,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
@@ -130,20 +107,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
@@ -162,22 +131,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let mut data =
let data =
tokio::task::spawn_blocking(move || parser::parse_komik_detail_document(&html))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
if !data.poster.is_empty() {
data.poster = get_cached_or_original(
self.db.clone(),
&self.redis_pool,
&data.poster,
self.semaphore.clone(),
)
.await;
}
Ok(data)
})
.await
@@ -196,7 +155,7 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let mut data = tokio::task::spawn_blocking({
let data = tokio::task::spawn_blocking({
let chapter_url = chapter_url.clone();
move || parser::parse_komik_chapter_document(&html, &chapter_url)
})
@@ -204,14 +163,6 @@ impl KomikUseCases {
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
data.images = cache_image_urls_batch_lazy(
self.db.clone(),
&self.redis_pool,
data.images,
self.semaphore.clone(),
)
.await;
Ok(data)
})
.await
@@ -278,7 +229,7 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
@@ -288,14 +239,6 @@ impl KomikUseCases {
return Err(format!("Empty komik {} page {}", list_name, page));
}
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
@@ -318,20 +261,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
@@ -354,20 +289,12 @@ impl KomikUseCases {
.await
.map_err(|e| e.to_string())?;
let (mut komik_list, pagination) =
let (komik_list, pagination) =
tokio::task::spawn_blocking(move || parser::parse_genre_page(&html, page))
.await
.map_err(|e| e.to_string())?
.map_err(|e| e.to_string())?;
apply_cached_posters(
&mut komik_list,
self.db.clone(),
&self.redis_pool,
self.semaphore.clone(),
)
.await;
Ok((komik_list, pagination))
})
.await
src/application/proxy/use_cases.rs
-235
View File
@@ -1,236 +1 @@
//! Proxy application use cases.
//!
//! Provides proxy fetch, image caching, and audit/repair operations.
//!
//! TODO: Move result types to `crate::presentation::dto::proxy`.
//! TODO: Add event bus integration for ImageRepaired events.
//! TODO: Replace `reqwest::Client::new()` with shared HTTP client from infrastructure.
use std::sync::Arc;
use axum::http::StatusCode;
use axum::response::Response;
use serde::Serialize;
use tracing::{error, info, warn};
use utoipa::ToSchema;
use crate::domain::error::*;
use crate::domain::repository::ImageCacheRepository;
use crate::infrastructure::repository::{ProxyRepository, SeaOrmImageCacheRepository};
use crate::infrastructure::services::images::cache::ImageCache;
// ============================================================================
// Result types — TEMPORARY: move to presentation layer
// ============================================================================
/// Result of caching an image URL.
/// TODO: Move to presentation::dto::proxy
#[derive(Debug, Serialize, ToSchema)]
pub struct ImageCacheResult {
pub success: bool,
pub original_url: String,
pub cdn_url: String,
pub from_cache: bool,
pub pending: Option<bool>,
}
/// Result of auditing/repairing a cached image URL.
/// TODO: Move to presentation::dto::proxy
#[derive(Debug, Serialize, ToSchema)]
pub struct AuditImageCacheResult {
pub success: bool,
pub original_url: String,
pub cdn_url: Option<String>,
pub was_accessible: bool,
pub re_uploaded: bool,
pub message: String,
}
// ============================================================================
// Use case struct
// ============================================================================
pub struct ProxyUseCases {
repository: ProxyRepository,
image_cache_repo: Arc<dyn ImageCacheRepository>,
}
impl ProxyUseCases {
pub fn new(
repository: ProxyRepository,
image_cache_repo: Arc<SeaOrmImageCacheRepository>,
) -> Self {
Self {
repository,
image_cache_repo,
}
}
fn build_image_cache(&self) -> ImageCache {
ImageCache::new(self.image_cache_repo.clone())
}
pub async fn fetch_with_proxy_only(&self, url: String) -> Result<Response, DomainError> {
let fetch_result = self
.repository
.fetch_with_proxy_url(&url)
.await
.map_err(|e| DomainError::Scraping(ScrapingError::Http(e.to_string())))?;
let mut builder = Response::builder().status(StatusCode::OK);
if let Some(content_type) = fetch_result.content_type {
builder = builder.header("Content-Type", content_type);
}
builder
.body(fetch_result.data.into())
.map_err(|e| DomainError::Repository(RepositoryError::Network(e.to_string())))
}
pub async fn image_cache(
&self,
url: String,
lazy: bool,
) -> Result<ImageCacheResult, DomainError> {
let cache = self.build_image_cache();
if let Some(cdn_url) = cache.get_cdn_url(&url).await {
return Ok(ImageCacheResult {
success: true,
original_url: url,
cdn_url,
from_cache: true,
pending: None,
});
}
if lazy {
let repo = self.image_cache_repo.clone();
let url_clone = url.clone();
tokio::spawn(async move {
let cache = ImageCache::new(repo);
match cache.get_or_cache(&url_clone).await {
Ok(cdn) => info!("[LazyCache] Cached {} -> {}", url_clone, cdn),
Err(e) => warn!("[LazyCache] Failed {}: {}", url_clone, e),
}
});
return Ok(ImageCacheResult {
success: true,
original_url: url.clone(),
cdn_url: url,
from_cache: false,
pending: Some(true),
});
}
match cache.get_or_cache(&url).await {
Ok(cdn_url) => Ok(ImageCacheResult {
success: true,
original_url: url,
cdn_url,
from_cache: false,
pending: None,
}),
Err(e) => {
error!("ImageCache error: {}", e);
Ok(ImageCacheResult {
success: false,
original_url: url.clone(),
cdn_url: url,
from_cache: false,
pending: None,
})
}
}
}
pub async fn audit_image_cache(
&self,
url: String,
) -> Result<AuditImageCacheResult, DomainError> {
let cache = self.build_image_cache();
let mut cdn_opt = cache.get_cdn_url(&url).await;
let mut original = url.clone();
if cdn_opt.is_none() {
if let Some(orig) = cache.find_original_from_cdn(&url).await {
info!("SmartAudit: {} recognized as CDN, original {}", url, orig);
original = orig;
cdn_opt = Some(url.clone());
}
}
if let Some(cdn_url) = cdn_opt {
let client = reqwest::Client::new();
let mut accessible = false;
match client.get(&cdn_url).send().await {
Ok(resp) if resp.status().is_success() => {
if let Ok(bytes) = resp.bytes().await {
if infer::get(&bytes)
.map(|k| k.mime_type().starts_with("image/"))
.unwrap_or(false)
{
accessible = true;
} else {
warn!("CDN {} returned non-image content", cdn_url);
}
}
}
Ok(resp) => warn!("CDN {} status {}", cdn_url, resp.status()),
Err(e) => warn!("CDN {} fetch error {}", cdn_url, e),
}
if accessible {
return Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(cdn_url),
was_accessible: true,
re_uploaded: false,
message: "CDN URL is accessible and the image is valid".to_string(),
});
}
info!("CDN {} inaccessible, purging and reuploading", cdn_url);
let _ = cache.invalidate(&original).await;
match cache.get_or_cache(&original).await {
Ok(new_cdn) => Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(new_cdn),
was_accessible: false,
re_uploaded: true,
message: "CDN URL was inaccessible, re-uploaded".to_string(),
}),
Err(e) => Ok(AuditImageCacheResult {
success: false,
original_url: original,
cdn_url: None,
was_accessible: false,
re_uploaded: false,
message: format!("Re-upload failed: {}", e),
}),
}
} else {
match cache.get_or_cache(&original).await {
Ok(new_cdn) => Ok(AuditImageCacheResult {
success: true,
original_url: original,
cdn_url: Some(new_cdn),
was_accessible: false,
re_uploaded: true,
message: "Cached newly".to_string(),
}),
Err(e) => Ok(AuditImageCacheResult {
success: false,
original_url: original,
cdn_url: None,
was_accessible: false,
re_uploaded: false,
message: format!("Cache failed: {}", e),
}),
}
}
}
}