vegardbm/manga-cli
0
1
Fork 0
Code Issues Pull Requests Releases Activity

fix bug where chapter objects were missing and improve manga client

Browse Source
This commit is contained in:
Vegard Bieker Matthey
2025-07-02 00:34:53 +02:00
parent 1a29600d88
commit e797a8b8ae
4 changed files with 234 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
src/client.rs
View File

@@ -1,11 +1,13 @@
use crate::response_deserializer::{Chapter, Id, Language};
use crate::response_deserializer::{Chapter, ChapterFeed, Id, Language, Manga};
use crate::util::{ConfigSearch, ConfigSelectionType, CoverSize, SelectionRange};
use crate::BASE;
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware};
type Client = ClientWithMiddleware;
pub struct MangaClientBuilder {
client: ClientWithMiddleware,
client: Client,
search_filter: Option<SearchFilter>,
bonus: Option<bool>,
result_limit: Option<u32>,
@@ -19,7 +21,7 @@ pub struct MangaClientBuilder {
#[derive(Debug)]
pub struct MangaClient {
pub client: ClientWithMiddleware,
pub client: Client,
pub search_filter: SearchFilter,
pub result_limit: u32,
pub cover_size: CoverSize,
@@ -183,9 +185,9 @@ impl MangaClientBuilder {
selection: crate::util::Selection::new(
match self.selection_type {
Some(a) => match a {
ConfigSelectionType::Chapter => crate::util::SelectionType::Chapter(
self.selection_range.unwrap(),
),
ConfigSelectionType::Chapter => {
crate::util::SelectionType::Chapter(self.selection_range.unwrap())
}
ConfigSelectionType::Volume => {
crate::util::SelectionType::Volume(self.selection_range.unwrap())
}
@@ -199,8 +201,6 @@ impl MangaClientBuilder {
}
}
use crate::response_deserializer::Manga;
impl MangaClient {
pub async fn get_manga(&self) -> Vec<Manga> {
match &self.search {
@@ -246,28 +246,37 @@ impl MangaClient {
}
pub async fn get_chapters(&self, id: &Id) -> Result<Vec<Chapter>, reqwest_middleware::Error> {
let limit = 50;
let limit = limit.to_string();
let limit = 100;
let limit_s = limit.to_string();
let lang = self.language.to_string();
let params = [
("limit", limit.as_str()),
("translatedLanguage[]", &self.language.to_string()),
("translatedLanguage[]", lang.as_str()),
("limit", limit_s.as_str()),
// These are apparently necessary, or else you may miss some chapters
("order[chapter]", "asc"),
("offset", "0"),
];
let url = format!("{BASE}/manga/{id}/feed");
let json = self
.client
.get(url)
.get(&url)
.query(&params)
.send()
.await?
.text()
.await?;
let mut result = crate::response_deserializer::deserialize_chapter_feed(&json).unwrap();
let mut chapters = Vec::new();
chapters.append(&mut result.data);
let mut total_chapters_received = result.limit;
while total_chapters_received < result.total {
let offset = total_chapters_received.to_string();
let params = [params[0], params[1], ("offset", offset.as_str())];
let url = format!("{BASE}/manga/{id}/feed");
let total = result.total;
let iters = total / limit + if total % limit != 0 { 1 } else { 0 };
futures::future::join_all((1..iters).map(|i| {
let url = &url;
async move {
let offset = (i * limit).to_string();
let params = [params[0], params[1], params[2], ("offset", offset.as_str())];
let json = self
.client
.get(url)
@@ -276,12 +285,75 @@ impl MangaClient {
.await?
.text()
.await?;
let mut new_result =
crate::response_deserializer::deserialize_chapter_feed(&json).unwrap();
result.data.append(&mut new_result.data);
total_chapters_received += result.limit;
Ok::<ChapterFeed, reqwest_middleware::Error>(
crate::response_deserializer::deserialize_chapter_feed(&json).unwrap(),
)
}
assert_eq!(result.data.len(), result.total as usize);
Ok(result.data)
}))
.await
.into_iter()
.collect::<Result<Vec<ChapterFeed>, reqwest_middleware::Error>>()?
.iter_mut()
.for_each(|m| chapters.append(&mut m.data));
assert_eq!(chapters.len(), total as usize);
Ok(chapters)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn ensure_getting_all_chapters() {
let client = MangaClientBuilder::new()
.selection_type(crate::util::ConfigSelectionType::Volume)
.selection_range(crate::util::SelectionRange::All)
.build();
// These tests should only be done for completed manga because last chapter might change
// otherwise
test_chapters_for_manga(
&client,
&Id(String::from("5a547d1d-576b-477f-8cb3-70a3b4187f8a")),
"JoJo's Bizarre Adventure Part 1 - Phantom Blood",
44,
)
.await;
test_chapters_for_manga(
&client,
&Id(String::from("0d545e62-d4cd-4e65-a65c-a5c46b794918")),
"JoJo's Bizarre Adventure Part 3 - Stardust Crusaders",
152,
)
.await;
test_chapters_for_manga(
&client,
&Id(String::from("319df2e2-e6a6-4e3a-a31c-68539c140a84")),
"Slam Dunk!",
276,
)
.await;
}
async fn test_chapters_for_manga(client: &MangaClient, id: &Id, title: &str, len: usize) {
let chapters = client.get_chapters(id).await.unwrap();
let mut test = vec![false; len];
for chapter in &chapters {
let n = chapter.attributes.chapter.unwrap();
if n.round() == n && n >= 1. {
test[n as usize - 1] = true;
}
}
for i in 0..test.len() {
if !test[i] {
println!("chapter {} not found for: {title}", i + 1);
}
}
assert!(!test.iter().any(|m| *m == false));
}
}

13
src/main.rs
View File

@@ -1,8 +1,7 @@
use client::{MangaClient, MangaClientBuilder, SearchFilter};
use response_deserializer::{Chapter, Id, Language, Manga};
use select::Entry;
use std::future::Future;
use std::pin::Pin;
use std::{future::Future, pin::Pin};
use util::SelectionType;
mod client;
@@ -12,7 +11,7 @@ mod select;
mod test;
mod util;
const BASE: &str = "https://api.mangadex.dev";
const BASE: &str = "https://api.mangadex.org";
#[tokio::main]
async fn main() {
@@ -22,7 +21,7 @@ async fn main() {
.bonus(config.bonus.unwrap_or_else(util::ask_bonus))
.cover(config.cover.unwrap_or_else(util::ask_cover))
.result_limit(config.result_limit.unwrap_or(5))
.cover_size(util::CoverSize::W512)
.cover_size(util::CoverSize::Full)
.selection_type(
config
.selection_type
@@ -40,6 +39,10 @@ async fn main() {
.language(Language::default())
.build();
let search_result = manga_client.get_manga().await;
if search_result.is_empty() {
eprintln!("Found no search results");
std::process::exit(1);
}
let mut choice = 0;
if search_result.len() > 1 {
@@ -69,7 +72,7 @@ async fn main() {
let title = &manga.attributes.title.en;
util::download_the_stuff(
util::download_selected_chapters(
&manga_client.client,
&selected_chapters,
title,

4
src/response_deserializer.rs
View File

@@ -580,8 +580,8 @@ pub struct Links {
#[derive(Deserialize, Debug)]
pub struct Description {
en: Option<String>,
ru: Option<String>,
pub en: Option<String>,
pub ru: Option<String>,
}
#[derive(Deserialize, Debug)]

167
src/util.rs
View File

@@ -1,12 +1,17 @@
use crate::error::{ChapterImageError, ChapterImagesError};
use crate::response_deserializer::ChapterImages;
use crate::Chapter;
use crate::BASE;
use crate::{Chapter, Id, BASE};
use icy_sixel::{DiffusionMethod, MethodForLargest, MethodForRep, PixelFormat, Quality};
use image::DynamicImage;
use reqwest_middleware::ClientWithMiddleware;
use std::{io, io::Write};
use std::{fs::File, io, io::Write, path::Path};
use zip::{
write::{SimpleFileOptions, ZipWriter},
CompressionMethod,
};
type Client = ClientWithMiddleware;
type Client = reqwest_middleware::ClientWithMiddleware;
const CONCURRENT_REQUESTS: usize = 20;
#[derive(Debug)]
pub struct Selection {
@@ -14,6 +19,13 @@ pub struct Selection {
pub bonus: bool, // Allows including or excluding bonus chapters and volumes
}
struct ImageItem {
url: String,
chapter: Option<f32>,
volume: Option<f32>,
index: usize,
}
impl Selection {
pub fn new(selection_type: SelectionType, bonus: bool) -> Self {
Self {
@@ -255,11 +267,7 @@ pub fn convert_to_sixel(data: &[u8]) -> String {
pixels.push(m.0[0]);
pixels.push(m.0[0]);
}),
DynamicImage::ImageLumaA8(_) => println!("Found lumaa8 image"),
DynamicImage::ImageRgb16(_) => println!("Found rgb16 image"),
DynamicImage::ImageLuma16(_) => println!("Found luma16 image"),
DynamicImage::ImageRgba16(_) => println!("Found rgba16 image"),
_ => panic!(),
_ => unimplemented!(),
}
icy_sixel::sixel_string(
&pixels,
@@ -269,7 +277,7 @@ pub fn convert_to_sixel(data: &[u8]) -> String {
DiffusionMethod::Auto,
MethodForLargest::Auto,
MethodForRep::Auto,
Quality::HIGH,
Quality::AUTO,
)
.unwrap()
}
@@ -295,15 +303,13 @@ impl Config {
}
}
use crate::Id;
use crate::error::{ChapterImageError, ChapterImagesError};
pub async fn download_the_stuff(
pub async fn download_selected_chapters(
client: &Client,
selected_chapters: &Vec<&Chapter>,
title: &str,
create_volumes: bool,
) {
let mut items = Vec::new();
for (i, chapter) in selected_chapters.iter().enumerate() {
let chapter_image_data = loop {
let json = client
@@ -342,26 +348,67 @@ pub async fn download_the_stuff(
"\x1b[1A\x1b[2Kdownloaded chapter json image data: [{i}/{}]",
selected_chapters.len()
);
let chapter =
download_chapter_images(client, &chapter_image_data, selected_chapters[i]).await;
match chapter {
Ok(chapter) => {
for (j, image) in chapter.iter().enumerate() {
let chapter_n = selected_chapters[i].attributes.chapter;
let chapter_text = if let Some(n) = chapter_n {
for n in construct_image_items(&chapter_image_data, chapter) {
items.push(async move {
let data = client
.clone()
.get(&n.url)
.send()
.await
.unwrap()
.bytes()
.await
.unwrap();
println!(
"\x1b[1A\x1b[2KDownloaded volume: {:?}, chapter: {:?}, title: {}, [{}/{}]",
chapter.attributes.volume,
chapter.attributes.chapter,
chapter.attributes.title,
n.index,
chapter.attributes.pages
);
(n, image::load_from_memory(&data).unwrap())
});
}
while items.len() >= CONCURRENT_REQUESTS {
let mut list = Vec::with_capacity(CONCURRENT_REQUESTS);
for _ in 0..CONCURRENT_REQUESTS {
list.push(items.pop().unwrap());
}
let complete = futures::future::join_all(list).await;
save_images(complete, title, create_volumes);
}
if i == selected_chapters.len() - 1 {
let mut list = Vec::with_capacity(items.len());
for _ in 0..items.len() {
list.push(items.pop().unwrap());
}
let complete = futures::future::join_all(list).await;
save_images(complete, title, create_volumes);
}
}
}
fn save_images(
image_data: Vec<(ImageItem, image::DynamicImage)>,
title: &str,
create_volumes: bool,
) {
for (n, image) in image_data {
let chapter_text = if let Some(n) = n.chapter {
n.to_string()
} else {
String::from("_none")
};
let chapter_path = format!(
"images/{}/chapter_{:0>3}_image_{:0>3}.png",
title, chapter_text, j
title, chapter_text, n.index
);
let path = if let Some(v) = selected_chapters[i].attributes.volume {
let path = if let Some(v) = n.volume {
if create_volumes {
format!(
"images/{}/volume_{:0>3}/chapter_{:0>3}_image_{:0>3}.png",
title, v, chapter_text, j
title, v, chapter_text, n.index
)
} else {
chapter_path
@@ -370,9 +417,7 @@ pub async fn download_the_stuff(
chapter_path
};
let path = std::path::Path::new(&path);
if selected_chapters[i].attributes.volume.is_some()
&& !&path.parent().unwrap().exists()
{
if n.volume.is_some() && !&path.parent().unwrap().exists() {
if create_volumes && !path.parent().unwrap().parent().unwrap().exists() {
std::fs::create_dir(path.parent().unwrap().parent().unwrap()).unwrap();
}
@@ -381,24 +426,12 @@ pub async fn download_the_stuff(
image.save(path).unwrap();
}
}
Err(e) => {
panic!("{e}");
}
}
}
}
pub fn create_volumes_or_chapters(
selected_chapters: &Vec<&Chapter>,
create_volumes: bool,
title: &str,
) {
use std::fs::File;
use std::io::Write;
use std::path::Path;
use zip::write::{SimpleFileOptions, ZipWriter};
use zip::CompressionMethod;
if create_volumes {
let mut volumes = Vec::new();
selected_chapters
@@ -452,54 +485,38 @@ pub fn create_volumes_or_chapters(
if entry.path().is_dir() {
continue;
}
if entry.file_name().to_str().unwrap()[..18]
== *format!("chapter_{:0>3}_image_", chapter).as_str()
{
zip.start_file(entry.file_name().to_str().unwrap(), options)
.unwrap();
let buffer = std::fs::read(entry.path()).unwrap();
zip.write_all(&buffer).unwrap();
}
}
zip.finish().unwrap();
}
}
}
async fn download_chapter_images(
client: &Client,
image_data: &ChapterImages,
chapter: &Chapter,
) -> Result<Vec<image::DynamicImage>, reqwest_middleware::Error> {
let mut data_futures = vec![];
for (i, file_name) in image_data.chapter.data.iter().enumerate() {
let base_url: &str = image_data.base_url.as_str();
let hash: &str = image_data.chapter.hash.as_str();
let future = async move {
let data = client
.clone()
.get(format!("{base_url}/data/{hash}/{file_name}"))
.send()
.await
.unwrap()
.bytes()
.await
.unwrap();
println!(
"\x1b[1A\x1b[2KDownloaded volume: {:?}, chapter: {:?}, title: {}, [{}/{}]",
chapter.attributes.volume,
chapter.attributes.chapter,
chapter.attributes.title,
i,
chapter.attributes.pages
);
data
};
data_futures.push(future);
}
Ok(futures::future::join_all(data_futures)
.await
fn construct_image_items(image_data: &ChapterImages, chapter: &Chapter) -> Vec<ImageItem> {
image_data
.chapter
.data
.iter()
.map(|m| image::load_from_memory(m).unwrap())
.collect())
.enumerate()
.map(|(i, m)| ImageItem {
url: format!(
"{}/data/{}/{m}",
image_data.base_url, image_data.chapter.hash
),
chapter: chapter.attributes.chapter,
volume: chapter.attributes.volume,
index: i,
})
.collect()
}
pub fn ask_cover() -> bool {