From e797a8b8aeb475c6d3037223168f985d5fa91e9d Mon Sep 17 00:00:00 2001 From: Vegard Matthey Date: Wed, 2 Jul 2025 00:34:53 +0200 Subject: [PATCH] fix bug where chapter objects were missing and improve manga client --- src/client.rs | 138 ++++++++++++++++------ src/main.rs | 13 ++- src/response_deserializer.rs | 4 +- src/util.rs | 221 +++++++++++++++++++---------------- 4 files changed, 234 insertions(+), 142 deletions(-) diff --git a/src/client.rs b/src/client.rs index 7df4e5b..4ce8e17 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,11 +1,13 @@ -use crate::response_deserializer::{Chapter, Id, Language}; +use crate::response_deserializer::{Chapter, ChapterFeed, Id, Language, Manga}; use crate::util::{ConfigSearch, ConfigSelectionType, CoverSize, SelectionRange}; use crate::BASE; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; +type Client = ClientWithMiddleware; + pub struct MangaClientBuilder { - client: ClientWithMiddleware, + client: Client, search_filter: Option, bonus: Option, result_limit: Option, @@ -19,7 +21,7 @@ pub struct MangaClientBuilder { #[derive(Debug)] pub struct MangaClient { - pub client: ClientWithMiddleware, + pub client: Client, pub search_filter: SearchFilter, pub result_limit: u32, pub cover_size: CoverSize, @@ -183,9 +185,9 @@ impl MangaClientBuilder { selection: crate::util::Selection::new( match self.selection_type { Some(a) => match a { - ConfigSelectionType::Chapter => crate::util::SelectionType::Chapter( - self.selection_range.unwrap(), - ), + ConfigSelectionType::Chapter => { + crate::util::SelectionType::Chapter(self.selection_range.unwrap()) + } ConfigSelectionType::Volume => { crate::util::SelectionType::Volume(self.selection_range.unwrap()) } @@ -199,8 +201,6 @@ impl MangaClientBuilder { } } -use crate::response_deserializer::Manga; - impl MangaClient { pub async fn get_manga(&self) -> Vec { match &self.search { @@ -246,42 +246,114 @@ impl MangaClient { } pub async fn get_chapters(&self, id: &Id) -> Result, reqwest_middleware::Error> { - let limit = 50; - let limit = limit.to_string(); + let limit = 100; + let limit_s = limit.to_string(); + let lang = self.language.to_string(); let params = [ - ("limit", limit.as_str()), - ("translatedLanguage[]", &self.language.to_string()), + ("translatedLanguage[]", lang.as_str()), + ("limit", limit_s.as_str()), + // These are apparently necessary, or else you may miss some chapters + ("order[chapter]", "asc"), + ("offset", "0"), ]; let url = format!("{BASE}/manga/{id}/feed"); let json = self .client - .get(url) + .get(&url) .query(¶ms) .send() .await? .text() .await?; let mut result = crate::response_deserializer::deserialize_chapter_feed(&json).unwrap(); + let mut chapters = Vec::new(); + chapters.append(&mut result.data); - let mut total_chapters_received = result.limit; - while total_chapters_received < result.total { - let offset = total_chapters_received.to_string(); - let params = [params[0], params[1], ("offset", offset.as_str())]; - let url = format!("{BASE}/manga/{id}/feed"); - let json = self - .client - .get(url) - .query(¶ms) - .send() - .await? - .text() - .await?; - let mut new_result = - crate::response_deserializer::deserialize_chapter_feed(&json).unwrap(); - result.data.append(&mut new_result.data); - total_chapters_received += result.limit; - } - assert_eq!(result.data.len(), result.total as usize); - Ok(result.data) + let total = result.total; + let iters = total / limit + if total % limit != 0 { 1 } else { 0 }; + + futures::future::join_all((1..iters).map(|i| { + let url = &url; + async move { + let offset = (i * limit).to_string(); + let params = [params[0], params[1], params[2], ("offset", offset.as_str())]; + let json = self + .client + .get(url) + .query(¶ms) + .send() + .await? + .text() + .await?; + Ok::( + crate::response_deserializer::deserialize_chapter_feed(&json).unwrap(), + ) + } + })) + .await + .into_iter() + .collect::, reqwest_middleware::Error>>()? + .iter_mut() + .for_each(|m| chapters.append(&mut m.data)); + + assert_eq!(chapters.len(), total as usize); + + Ok(chapters) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn ensure_getting_all_chapters() { + let client = MangaClientBuilder::new() + .selection_type(crate::util::ConfigSelectionType::Volume) + .selection_range(crate::util::SelectionRange::All) + .build(); + + // These tests should only be done for completed manga because last chapter might change + // otherwise + test_chapters_for_manga( + &client, + &Id(String::from("5a547d1d-576b-477f-8cb3-70a3b4187f8a")), + "JoJo's Bizarre Adventure Part 1 - Phantom Blood", + 44, + ) + .await; + + test_chapters_for_manga( + &client, + &Id(String::from("0d545e62-d4cd-4e65-a65c-a5c46b794918")), + "JoJo's Bizarre Adventure Part 3 - Stardust Crusaders", + 152, + ) + .await; + + test_chapters_for_manga( + &client, + &Id(String::from("319df2e2-e6a6-4e3a-a31c-68539c140a84")), + "Slam Dunk!", + 276, + ) + .await; + } + + async fn test_chapters_for_manga(client: &MangaClient, id: &Id, title: &str, len: usize) { + let chapters = client.get_chapters(id).await.unwrap(); + let mut test = vec![false; len]; + for chapter in &chapters { + let n = chapter.attributes.chapter.unwrap(); + if n.round() == n && n >= 1. { + test[n as usize - 1] = true; + } + } + for i in 0..test.len() { + if !test[i] { + println!("chapter {} not found for: {title}", i + 1); + } + } + assert!(!test.iter().any(|m| *m == false)); } } diff --git a/src/main.rs b/src/main.rs index 175471e..7a5b571 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,7 @@ use client::{MangaClient, MangaClientBuilder, SearchFilter}; use response_deserializer::{Chapter, Id, Language, Manga}; use select::Entry; -use std::future::Future; -use std::pin::Pin; +use std::{future::Future, pin::Pin}; use util::SelectionType; mod client; @@ -12,7 +11,7 @@ mod select; mod test; mod util; -const BASE: &str = "https://api.mangadex.dev"; +const BASE: &str = "https://api.mangadex.org"; #[tokio::main] async fn main() { @@ -22,7 +21,7 @@ async fn main() { .bonus(config.bonus.unwrap_or_else(util::ask_bonus)) .cover(config.cover.unwrap_or_else(util::ask_cover)) .result_limit(config.result_limit.unwrap_or(5)) - .cover_size(util::CoverSize::W512) + .cover_size(util::CoverSize::Full) .selection_type( config .selection_type @@ -40,6 +39,10 @@ async fn main() { .language(Language::default()) .build(); let search_result = manga_client.get_manga().await; + if search_result.is_empty() { + eprintln!("Found no search results"); + std::process::exit(1); + } let mut choice = 0; if search_result.len() > 1 { @@ -69,7 +72,7 @@ async fn main() { let title = &manga.attributes.title.en; - util::download_the_stuff( + util::download_selected_chapters( &manga_client.client, &selected_chapters, title, diff --git a/src/response_deserializer.rs b/src/response_deserializer.rs index 4b45844..b85c4d1 100644 --- a/src/response_deserializer.rs +++ b/src/response_deserializer.rs @@ -580,8 +580,8 @@ pub struct Links { #[derive(Deserialize, Debug)] pub struct Description { - en: Option, - ru: Option, + pub en: Option, + pub ru: Option, } #[derive(Deserialize, Debug)] diff --git a/src/util.rs b/src/util.rs index 4514f03..b0839d4 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,12 +1,17 @@ +use crate::error::{ChapterImageError, ChapterImagesError}; use crate::response_deserializer::ChapterImages; -use crate::Chapter; -use crate::BASE; +use crate::{Chapter, Id, BASE}; use icy_sixel::{DiffusionMethod, MethodForLargest, MethodForRep, PixelFormat, Quality}; use image::DynamicImage; -use reqwest_middleware::ClientWithMiddleware; -use std::{io, io::Write}; +use std::{fs::File, io, io::Write, path::Path}; +use zip::{ + write::{SimpleFileOptions, ZipWriter}, + CompressionMethod, +}; -type Client = ClientWithMiddleware; +type Client = reqwest_middleware::ClientWithMiddleware; + +const CONCURRENT_REQUESTS: usize = 20; #[derive(Debug)] pub struct Selection { @@ -14,6 +19,13 @@ pub struct Selection { pub bonus: bool, // Allows including or excluding bonus chapters and volumes } +struct ImageItem { + url: String, + chapter: Option, + volume: Option, + index: usize, +} + impl Selection { pub fn new(selection_type: SelectionType, bonus: bool) -> Self { Self { @@ -255,11 +267,7 @@ pub fn convert_to_sixel(data: &[u8]) -> String { pixels.push(m.0[0]); pixels.push(m.0[0]); }), - DynamicImage::ImageLumaA8(_) => println!("Found lumaa8 image"), - DynamicImage::ImageRgb16(_) => println!("Found rgb16 image"), - DynamicImage::ImageLuma16(_) => println!("Found luma16 image"), - DynamicImage::ImageRgba16(_) => println!("Found rgba16 image"), - _ => panic!(), + _ => unimplemented!(), } icy_sixel::sixel_string( &pixels, @@ -269,7 +277,7 @@ pub fn convert_to_sixel(data: &[u8]) -> String { DiffusionMethod::Auto, MethodForLargest::Auto, MethodForRep::Auto, - Quality::HIGH, + Quality::AUTO, ) .unwrap() } @@ -295,15 +303,13 @@ impl Config { } } -use crate::Id; - -use crate::error::{ChapterImageError, ChapterImagesError}; -pub async fn download_the_stuff( +pub async fn download_selected_chapters( client: &Client, selected_chapters: &Vec<&Chapter>, title: &str, create_volumes: bool, ) { + let mut items = Vec::new(); for (i, chapter) in selected_chapters.iter().enumerate() { let chapter_image_data = loop { let json = client @@ -342,49 +348,82 @@ pub async fn download_the_stuff( "\x1b[1A\x1b[2Kdownloaded chapter json image data: [{i}/{}]", selected_chapters.len() ); - let chapter = - download_chapter_images(client, &chapter_image_data, selected_chapters[i]).await; - match chapter { - Ok(chapter) => { - for (j, image) in chapter.iter().enumerate() { - let chapter_n = selected_chapters[i].attributes.chapter; - let chapter_text = if let Some(n) = chapter_n { - n.to_string() - } else { - String::from("_none") - }; - let chapter_path = format!( - "images/{}/chapter_{:0>3}_image_{:0>3}.png", - title, chapter_text, j - ); - let path = if let Some(v) = selected_chapters[i].attributes.volume { - if create_volumes { - format!( - "images/{}/volume_{:0>3}/chapter_{:0>3}_image_{:0>3}.png", - title, v, chapter_text, j - ) - } else { - chapter_path - } - } else { - chapter_path - }; - let path = std::path::Path::new(&path); - if selected_chapters[i].attributes.volume.is_some() - && !&path.parent().unwrap().exists() - { - if create_volumes && !path.parent().unwrap().parent().unwrap().exists() { - std::fs::create_dir(path.parent().unwrap().parent().unwrap()).unwrap(); - } - std::fs::create_dir(path.parent().unwrap()).unwrap(); - } - image.save(path).unwrap(); - } - } - Err(e) => { - panic!("{e}"); - } + for n in construct_image_items(&chapter_image_data, chapter) { + items.push(async move { + let data = client + .clone() + .get(&n.url) + .send() + .await + .unwrap() + .bytes() + .await + .unwrap(); + println!( + "\x1b[1A\x1b[2KDownloaded volume: {:?}, chapter: {:?}, title: {}, [{}/{}]", + chapter.attributes.volume, + chapter.attributes.chapter, + chapter.attributes.title, + n.index, + chapter.attributes.pages + ); + (n, image::load_from_memory(&data).unwrap()) + }); } + while items.len() >= CONCURRENT_REQUESTS { + let mut list = Vec::with_capacity(CONCURRENT_REQUESTS); + for _ in 0..CONCURRENT_REQUESTS { + list.push(items.pop().unwrap()); + } + let complete = futures::future::join_all(list).await; + save_images(complete, title, create_volumes); + } + if i == selected_chapters.len() - 1 { + let mut list = Vec::with_capacity(items.len()); + for _ in 0..items.len() { + list.push(items.pop().unwrap()); + } + let complete = futures::future::join_all(list).await; + save_images(complete, title, create_volumes); + } + } +} + +fn save_images( + image_data: Vec<(ImageItem, image::DynamicImage)>, + title: &str, + create_volumes: bool, +) { + for (n, image) in image_data { + let chapter_text = if let Some(n) = n.chapter { + n.to_string() + } else { + String::from("_none") + }; + let chapter_path = format!( + "images/{}/chapter_{:0>3}_image_{:0>3}.png", + title, chapter_text, n.index + ); + let path = if let Some(v) = n.volume { + if create_volumes { + format!( + "images/{}/volume_{:0>3}/chapter_{:0>3}_image_{:0>3}.png", + title, v, chapter_text, n.index + ) + } else { + chapter_path + } + } else { + chapter_path + }; + let path = std::path::Path::new(&path); + if n.volume.is_some() && !&path.parent().unwrap().exists() { + if create_volumes && !path.parent().unwrap().parent().unwrap().exists() { + std::fs::create_dir(path.parent().unwrap().parent().unwrap()).unwrap(); + } + std::fs::create_dir(path.parent().unwrap()).unwrap(); + } + image.save(path).unwrap(); } } @@ -393,12 +432,6 @@ pub fn create_volumes_or_chapters( create_volumes: bool, title: &str, ) { - use std::fs::File; - use std::io::Write; - use std::path::Path; - use zip::write::{SimpleFileOptions, ZipWriter}; - use zip::CompressionMethod; - if create_volumes { let mut volumes = Vec::new(); selected_chapters @@ -452,11 +485,15 @@ pub fn create_volumes_or_chapters( if entry.path().is_dir() { continue; } - zip.start_file(entry.file_name().to_str().unwrap(), options) - .unwrap(); + if entry.file_name().to_str().unwrap()[..18] + == *format!("chapter_{:0>3}_image_", chapter).as_str() + { + zip.start_file(entry.file_name().to_str().unwrap(), options) + .unwrap(); - let buffer = std::fs::read(entry.path()).unwrap(); - zip.write_all(&buffer).unwrap(); + let buffer = std::fs::read(entry.path()).unwrap(); + zip.write_all(&buffer).unwrap(); + } } zip.finish().unwrap(); @@ -464,42 +501,22 @@ pub fn create_volumes_or_chapters( } } -async fn download_chapter_images( - client: &Client, - image_data: &ChapterImages, - chapter: &Chapter, -) -> Result, reqwest_middleware::Error> { - let mut data_futures = vec![]; - for (i, file_name) in image_data.chapter.data.iter().enumerate() { - let base_url: &str = image_data.base_url.as_str(); - let hash: &str = image_data.chapter.hash.as_str(); - let future = async move { - let data = client - .clone() - .get(format!("{base_url}/data/{hash}/{file_name}")) - .send() - .await - .unwrap() - .bytes() - .await - .unwrap(); - println!( - "\x1b[1A\x1b[2KDownloaded volume: {:?}, chapter: {:?}, title: {}, [{}/{}]", - chapter.attributes.volume, - chapter.attributes.chapter, - chapter.attributes.title, - i, - chapter.attributes.pages - ); - data - }; - data_futures.push(future); - } - Ok(futures::future::join_all(data_futures) - .await +fn construct_image_items(image_data: &ChapterImages, chapter: &Chapter) -> Vec { + image_data + .chapter + .data .iter() - .map(|m| image::load_from_memory(m).unwrap()) - .collect()) + .enumerate() + .map(|(i, m)| ImageItem { + url: format!( + "{}/data/{}/{m}", + image_data.base_url, image_data.chapter.hash + ), + chapter: chapter.attributes.chapter, + volume: chapter.attributes.volume, + index: i, + }) + .collect() } pub fn ask_cover() -> bool {