fix bug where chapter objects were missing and improve manga client

src/client.rs

@@ -1,11 +1,13 @@
-use crate::response_deserializer::{Chapter, Id, Language};
+use crate::response_deserializer::{Chapter, ChapterFeed, Id, Language, Manga};
 use crate::util::{ConfigSearch, ConfigSelectionType, CoverSize, SelectionRange};
 use crate::BASE;
 use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
 use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware};
 
+type Client = ClientWithMiddleware;
+
 pub struct MangaClientBuilder {
-    client: ClientWithMiddleware,
+    client: Client,
     search_filter: Option<SearchFilter>,
     bonus: Option<bool>,
     result_limit: Option<u32>,
@@ -19,7 +21,7 @@ pub struct MangaClientBuilder {
 
 #[derive(Debug)]
 pub struct MangaClient {
-    pub client: ClientWithMiddleware,
+    pub client: Client,
     pub search_filter: SearchFilter,
     pub result_limit: u32,
     pub cover_size: CoverSize,
@@ -183,9 +185,9 @@ impl MangaClientBuilder {
             selection: crate::util::Selection::new(
                 match self.selection_type {
                     Some(a) => match a {
-                        ConfigSelectionType::Chapter => crate::util::SelectionType::Chapter(
+                        ConfigSelectionType::Chapter => {
-                            self.selection_range.unwrap(),
+                            crate::util::SelectionType::Chapter(self.selection_range.unwrap())
-                        ),
+                        }
                         ConfigSelectionType::Volume => {
                             crate::util::SelectionType::Volume(self.selection_range.unwrap())
                         }
@@ -199,8 +201,6 @@ impl MangaClientBuilder {
     }
 }
 
-use crate::response_deserializer::Manga;
-
 impl MangaClient {
     pub async fn get_manga(&self) -> Vec<Manga> {
         match &self.search {
@@ -246,28 +246,37 @@ impl MangaClient {
     }
 
     pub async fn get_chapters(&self, id: &Id) -> Result<Vec<Chapter>, reqwest_middleware::Error> {
-        let limit = 50;
+        let limit = 100;
-        let limit = limit.to_string();
+        let limit_s = limit.to_string();
+        let lang = self.language.to_string();
         let params = [
-            ("limit", limit.as_str()),
+            ("translatedLanguage[]", lang.as_str()),
-            ("translatedLanguage[]", &self.language.to_string()),
+            ("limit", limit_s.as_str()),
+            // These are apparently necessary, or else you may miss some chapters
+            ("order[chapter]", "asc"),
+            ("offset", "0"),
         ];
         let url = format!("{BASE}/manga/{id}/feed");
         let json = self
             .client
-            .get(url)
+            .get(&url)
             .query(&params)
             .send()
             .await?
             .text()
             .await?;
         let mut result = crate::response_deserializer::deserialize_chapter_feed(&json).unwrap();
+        let mut chapters = Vec::new();
+        chapters.append(&mut result.data);
 
-        let mut total_chapters_received = result.limit;
+        let total = result.total;
-        while total_chapters_received < result.total {
+        let iters = total / limit + if total % limit != 0 { 1 } else { 0 };
-            let offset = total_chapters_received.to_string();
+
-            let params = [params[0], params[1], ("offset", offset.as_str())];
+        futures::future::join_all((1..iters).map(|i| {
-            let url = format!("{BASE}/manga/{id}/feed");
+            let url = &url;
+            async move {
+                let offset = (i * limit).to_string();
+                let params = [params[0], params[1], params[2], ("offset", offset.as_str())];
                 let json = self
                     .client
                     .get(url)
@@ -276,12 +285,75 @@ impl MangaClient {
                     .await?
                     .text()
                     .await?;
-            let mut new_result =
+                Ok::<ChapterFeed, reqwest_middleware::Error>(
-                crate::response_deserializer::deserialize_chapter_feed(&json).unwrap();
+                    crate::response_deserializer::deserialize_chapter_feed(&json).unwrap(),
-            result.data.append(&mut new_result.data);
+                )
-            total_chapters_received += result.limit;
             }
-        assert_eq!(result.data.len(), result.total as usize);
+        }))
-        Ok(result.data)
+        .await
+        .into_iter()
+        .collect::<Result<Vec<ChapterFeed>, reqwest_middleware::Error>>()?
+        .iter_mut()
+        .for_each(|m| chapters.append(&mut m.data));
+
+        assert_eq!(chapters.len(), total as usize);
+
+        Ok(chapters)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn ensure_getting_all_chapters() {
+        let client = MangaClientBuilder::new()
+            .selection_type(crate::util::ConfigSelectionType::Volume)
+            .selection_range(crate::util::SelectionRange::All)
+            .build();
+
+        // These tests should only be done for completed manga because last chapter might change
+        // otherwise
+        test_chapters_for_manga(
+            &client,
+            &Id(String::from("5a547d1d-576b-477f-8cb3-70a3b4187f8a")),
+            "JoJo's Bizarre Adventure Part 1 - Phantom Blood",
+            44,
+        )
+        .await;
+
+        test_chapters_for_manga(
+            &client,
+            &Id(String::from("0d545e62-d4cd-4e65-a65c-a5c46b794918")),
+            "JoJo's Bizarre Adventure Part 3 - Stardust Crusaders",
+            152,
+        )
+        .await;
+
+        test_chapters_for_manga(
+            &client,
+            &Id(String::from("319df2e2-e6a6-4e3a-a31c-68539c140a84")),
+            "Slam Dunk!",
+            276,
+        )
+        .await;
+    }
+
+    async fn test_chapters_for_manga(client: &MangaClient, id: &Id, title: &str, len: usize) {
+        let chapters = client.get_chapters(id).await.unwrap();
+        let mut test = vec![false; len];
+        for chapter in &chapters {
+            let n = chapter.attributes.chapter.unwrap();
+            if n.round() == n && n >= 1. {
+                test[n as usize - 1] = true;
+            }
+        }
+        for i in 0..test.len() {
+            if !test[i] {
+                println!("chapter {} not found for: {title}", i + 1);
+            }
+        }
+        assert!(!test.iter().any(|m| *m == false));
     }
 }

src/main.rs

@@ -1,8 +1,7 @@
 use client::{MangaClient, MangaClientBuilder, SearchFilter};
 use response_deserializer::{Chapter, Id, Language, Manga};
 use select::Entry;
-use std::future::Future;
+use std::{future::Future, pin::Pin};
-use std::pin::Pin;
 use util::SelectionType;
 
 mod client;
@@ -12,7 +11,7 @@ mod select;
 mod test;
 mod util;
 
-const BASE: &str = "https://api.mangadex.dev";
+const BASE: &str = "https://api.mangadex.org";
 
 #[tokio::main]
 async fn main() {
@@ -22,7 +21,7 @@ async fn main() {
             .bonus(config.bonus.unwrap_or_else(util::ask_bonus))
             .cover(config.cover.unwrap_or_else(util::ask_cover))
             .result_limit(config.result_limit.unwrap_or(5))
-            .cover_size(util::CoverSize::W512)
+            .cover_size(util::CoverSize::Full)
             .selection_type(
                 config
                     .selection_type
@@ -40,6 +39,10 @@ async fn main() {
             .language(Language::default())
             .build();
     let search_result = manga_client.get_manga().await;
+    if search_result.is_empty() {
+        eprintln!("Found no search results");
+        std::process::exit(1);
+    }
 
     let mut choice = 0;
     if search_result.len() > 1 {
@@ -69,7 +72,7 @@ async fn main() {
 
     let title = &manga.attributes.title.en;
 
-    util::download_the_stuff(
+    util::download_selected_chapters(
         &manga_client.client,
         &selected_chapters,
         title,

src/response_deserializer.rs

@@ -580,8 +580,8 @@ pub struct Links {
 
 #[derive(Deserialize, Debug)]
 pub struct Description {
-    en: Option<String>,
+    pub en: Option<String>,
-    ru: Option<String>,
+    pub ru: Option<String>,
 }
 
 #[derive(Deserialize, Debug)]

src/util.rs

@@ -1,12 +1,17 @@
+use crate::error::{ChapterImageError, ChapterImagesError};
 use crate::response_deserializer::ChapterImages;
-use crate::Chapter;
+use crate::{Chapter, Id, BASE};
-use crate::BASE;
 use icy_sixel::{DiffusionMethod, MethodForLargest, MethodForRep, PixelFormat, Quality};
 use image::DynamicImage;
-use reqwest_middleware::ClientWithMiddleware;
+use std::{fs::File, io, io::Write, path::Path};
-use std::{io, io::Write};
+use zip::{
+    write::{SimpleFileOptions, ZipWriter},
+    CompressionMethod,
+};
 
-type Client = ClientWithMiddleware;
+type Client = reqwest_middleware::ClientWithMiddleware;
+
+const CONCURRENT_REQUESTS: usize = 20;
 
 #[derive(Debug)]
 pub struct Selection {
@@ -14,6 +19,13 @@ pub struct Selection {
     pub bonus: bool, // Allows including or excluding bonus chapters and volumes
 }
 
+struct ImageItem {
+    url: String,
+    chapter: Option<f32>,
+    volume: Option<f32>,
+    index: usize,
+}
+
 impl Selection {
     pub fn new(selection_type: SelectionType, bonus: bool) -> Self {
         Self {
@@ -255,11 +267,7 @@ pub fn convert_to_sixel(data: &[u8]) -> String {
             pixels.push(m.0[0]);
             pixels.push(m.0[0]);
         }),
-        DynamicImage::ImageLumaA8(_) => println!("Found lumaa8 image"),
+        _ => unimplemented!(),
-        DynamicImage::ImageRgb16(_) => println!("Found rgb16 image"),
-        DynamicImage::ImageLuma16(_) => println!("Found luma16 image"),
-        DynamicImage::ImageRgba16(_) => println!("Found rgba16 image"),
-        _ => panic!(),
     }
     icy_sixel::sixel_string(
         &pixels,
@@ -269,7 +277,7 @@ pub fn convert_to_sixel(data: &[u8]) -> String {
         DiffusionMethod::Auto,
         MethodForLargest::Auto,
         MethodForRep::Auto,
-        Quality::HIGH,
+        Quality::AUTO,
     )
     .unwrap()
 }
@@ -295,15 +303,13 @@ impl Config {
     }
 }
 
-use crate::Id;
+pub async fn download_selected_chapters(
-
-use crate::error::{ChapterImageError, ChapterImagesError};
-pub async fn download_the_stuff(
     client: &Client,
     selected_chapters: &Vec<&Chapter>,
     title: &str,
     create_volumes: bool,
 ) {
+    let mut items = Vec::new();
     for (i, chapter) in selected_chapters.iter().enumerate() {
         let chapter_image_data = loop {
             let json = client
@@ -342,26 +348,67 @@ pub async fn download_the_stuff(
             "\x1b[1A\x1b[2Kdownloaded chapter json image data: [{i}/{}]",
             selected_chapters.len()
         );
-        let chapter =
+        for n in construct_image_items(&chapter_image_data, chapter) {
-            download_chapter_images(client, &chapter_image_data, selected_chapters[i]).await;
+            items.push(async move {
-        match chapter {
+                let data = client
-            Ok(chapter) => {
+                    .clone()
-                for (j, image) in chapter.iter().enumerate() {
+                    .get(&n.url)
-                    let chapter_n = selected_chapters[i].attributes.chapter;
+                    .send()
-                    let chapter_text = if let Some(n) = chapter_n {
+                    .await
+                    .unwrap()
+                    .bytes()
+                    .await
+                    .unwrap();
+                println!(
+                    "\x1b[1A\x1b[2KDownloaded volume: {:?}, chapter: {:?}, title: {}, [{}/{}]",
+                    chapter.attributes.volume,
+                    chapter.attributes.chapter,
+                    chapter.attributes.title,
+                    n.index,
+                    chapter.attributes.pages
+                );
+                (n, image::load_from_memory(&data).unwrap())
+            });
+        }
+        while items.len() >= CONCURRENT_REQUESTS {
+            let mut list = Vec::with_capacity(CONCURRENT_REQUESTS);
+            for _ in 0..CONCURRENT_REQUESTS {
+                list.push(items.pop().unwrap());
+            }
+            let complete = futures::future::join_all(list).await;
+            save_images(complete, title, create_volumes);
+        }
+        if i == selected_chapters.len() - 1 {
+            let mut list = Vec::with_capacity(items.len());
+            for _ in 0..items.len() {
+                list.push(items.pop().unwrap());
+            }
+            let complete = futures::future::join_all(list).await;
+            save_images(complete, title, create_volumes);
+        }
+    }
+}
+
+fn save_images(
+    image_data: Vec<(ImageItem, image::DynamicImage)>,
+    title: &str,
+    create_volumes: bool,
+) {
+    for (n, image) in image_data {
+        let chapter_text = if let Some(n) = n.chapter {
             n.to_string()
         } else {
             String::from("_none")
         };
         let chapter_path = format!(
             "images/{}/chapter_{:0>3}_image_{:0>3}.png",
-                        title, chapter_text, j
+            title, chapter_text, n.index
         );
-                    let path = if let Some(v) = selected_chapters[i].attributes.volume {
+        let path = if let Some(v) = n.volume {
             if create_volumes {
                 format!(
                     "images/{}/volume_{:0>3}/chapter_{:0>3}_image_{:0>3}.png",
-                                title, v, chapter_text, j
+                    title, v, chapter_text, n.index
                 )
             } else {
                 chapter_path
@@ -370,9 +417,7 @@ pub async fn download_the_stuff(
             chapter_path
         };
         let path = std::path::Path::new(&path);
-                    if selected_chapters[i].attributes.volume.is_some()
+        if n.volume.is_some() && !&path.parent().unwrap().exists() {
-                        && !&path.parent().unwrap().exists()
-                    {
             if create_volumes && !path.parent().unwrap().parent().unwrap().exists() {
                 std::fs::create_dir(path.parent().unwrap().parent().unwrap()).unwrap();
             }
@@ -381,24 +426,12 @@ pub async fn download_the_stuff(
         image.save(path).unwrap();
     }
 }
-            Err(e) => {
-                panic!("{e}");
-            }
-        }
-    }
-}
 
 pub fn create_volumes_or_chapters(
     selected_chapters: &Vec<&Chapter>,
     create_volumes: bool,
     title: &str,
 ) {
-    use std::fs::File;
-    use std::io::Write;
-    use std::path::Path;
-    use zip::write::{SimpleFileOptions, ZipWriter};
-    use zip::CompressionMethod;
-
     if create_volumes {
         let mut volumes = Vec::new();
         selected_chapters
@@ -452,54 +485,38 @@ pub fn create_volumes_or_chapters(
                 if entry.path().is_dir() {
                     continue;
                 }
+                if entry.file_name().to_str().unwrap()[..18]
+                    == *format!("chapter_{:0>3}_image_", chapter).as_str()
+                {
                     zip.start_file(entry.file_name().to_str().unwrap(), options)
                         .unwrap();
 
                     let buffer = std::fs::read(entry.path()).unwrap();
                     zip.write_all(&buffer).unwrap();
                 }
+            }
 
             zip.finish().unwrap();
         }
     }
 }
 
-async fn download_chapter_images(
+fn construct_image_items(image_data: &ChapterImages, chapter: &Chapter) -> Vec<ImageItem> {
-    client: &Client,
+    image_data
-    image_data: &ChapterImages,
+        .chapter
-    chapter: &Chapter,
+        .data
-) -> Result<Vec<image::DynamicImage>, reqwest_middleware::Error> {
-    let mut data_futures = vec![];
-    for (i, file_name) in image_data.chapter.data.iter().enumerate() {
-        let base_url: &str = image_data.base_url.as_str();
-        let hash: &str = image_data.chapter.hash.as_str();
-        let future = async move {
-            let data = client
-                .clone()
-                .get(format!("{base_url}/data/{hash}/{file_name}"))
-                .send()
-                .await
-                .unwrap()
-                .bytes()
-                .await
-                .unwrap();
-            println!(
-                "\x1b[1A\x1b[2KDownloaded volume: {:?}, chapter: {:?}, title: {}, [{}/{}]",
-                chapter.attributes.volume,
-                chapter.attributes.chapter,
-                chapter.attributes.title,
-                i,
-                chapter.attributes.pages
-            );
-            data
-        };
-        data_futures.push(future);
-    }
-    Ok(futures::future::join_all(data_futures)
-        .await
         .iter()
-        .map(|m| image::load_from_memory(m).unwrap())
+        .enumerate()
-        .collect())
+        .map(|(i, m)| ImageItem {
+            url: format!(
+                "{}/data/{}/{m}",
+                image_data.base_url, image_data.chapter.hash
+            ),
+            chapter: chapter.attributes.chapter,
+            volume: chapter.attributes.volume,
+            index: i,
+        })
+        .collect()
 }
 
 pub fn ask_cover() -> bool {