diff --git a/Makefile b/Makefile index 85aa5b5..45e084d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .DEFAULT_GOAL := main.pdf main.pdf: main.tex jishoScrape/index.js title/titlepage.tex $(wildcard data/pages/*) $(wildcard data/tables/*) - for i in $$(seq 5 -1 1); do node jishoScrape/index.js n$$i; done + for i in $$(seq 7); do node jishoScrape/index.js grade$$i; done xelatex main.tex #TODO: Make this modular, fix the index.js target. diff --git a/jishoScrape/index.js b/jishoScrape/index.js index c5f21e4..02e6fdd 100644 --- a/jishoScrape/index.js +++ b/jishoScrape/index.js @@ -1,25 +1,26 @@ const fs = require('fs'); /* Import local files */ -const {fetchKanjiFromTxt, fetchJishoBufferData, fetchKanjiFromJisho} = require('./src/dataFetching.js'); -const {getKanjiTexData} = require('./src/texConversion.js'); -const {chapterTabular} = require('./src/kanjiTables.js'); - -function log(message, jlptLevel) { - const jlptLevelCaps = jlptLevel.toUpperCase(); - console.log(`${jlptLevelCaps}: ${message}`); -} +const { fetchJishoResults } = require('./src/dataFetching.js'); +const { getKanjiTexData } = require('./src/texConversion.js'); +const { chapterTabular } = require('./src/kanjiTables.js'); /* Encapsulate main process in async function */ -async function main(jlptLevel) { +async function main(grade) { - const jishoResults = await fetchJishoResults(jlptLevel); + /* Custom log function */ + function log(message) { + const gradeCaps = grade.toUpperCase(); + console.log(`${gradeCaps}: ${message}`); + } + + const jishoResults = await fetchJishoResults(grade, log); const kanjiArray = jishoResults.map(result => result.query); - log('Generating tex pages', jlptLevel); + log('Generating tex pages'); const texData = getKanjiTexData(jishoResults); - log('Generating chapter table page', jlptLevel); + log('Generating chapter table page'); const chapterTable = chapterTabular(kanjiArray, 16); let resultPage = ''; @@ -32,39 +33,23 @@ async function main(jlptLevel) { \\newpage\n`; } - fs.writeFile(`./data/tables/${jlptLevel}.tex`, chapterTable, (err) => {if (err) console.error(err)}); - fs.writeFile(`./data/pages/${jlptLevel}.tex`, resultPage, (err) => {if (err) console.error(err)}); -} - - /** Fetch data from buffer if available. - * Else fetch data from txt and jisho requests, - * and make buffer files - */ -async function fetchJishoResults(jlptLevel) { - - const bufferFileExists = fs.existsSync(`./data/jisho/${jlptLevel}.json`); - - if(bufferFileExists) { - log('Fetching Jisho data from buffer', jlptLevel) - return await fetchJishoBufferData(`./data/jisho/${jlptLevel}.json`); - } else { - log('Fetching data from Jisho and writing to buffer', jlptLevel) - return await fetchJishoDataAndWriteToBuffer(jlptLevel); - } -} - -async function fetchJishoDataAndWriteToBuffer(jlptLevel) { - const kanjiArray = await fetchKanjiFromTxt(`./data/txt/${jlptLevel}.txt`); - const jishoResults = await fetchKanjiFromJisho(kanjiArray); - fs.writeFile(`./data/jisho/${jlptLevel}.json`, JSON.stringify(jishoResults, null, " "), (err) => {if (err) console.error(err)}); - return jishoResults; + fs.writeFile( + `./data/tables/${grade}.tex`, + chapterTable, + (err) => { if (err) console.error(err) } + ); + fs.writeFile( + `./data/pages/${grade}.tex`, + resultPage, + (err) => { if (err) console.error(err) } + ); } /* Handle args */ async function argWrapper() { try { - if (!/n\d/.test(process.argv[2])) throw 'Input not valid'; + if (!/grade\d/.test(process.argv[2])) throw 'Input not valid'; await main(process.argv[2]); } catch (error) { console.error(error); diff --git a/jishoScrape/src/dataFetching.js b/jishoScrape/src/dataFetching.js index 8c2aba2..6bf3703 100644 --- a/jishoScrape/src/dataFetching.js +++ b/jishoScrape/src/dataFetching.js @@ -3,56 +3,67 @@ const util = require('util'); const jishoApi = require('unofficial-jisho-api'); const jisho = new jishoApi(); -/** - * Reads a txt file and splits the characters into an array - * @param {string} file Path to file - * @returns {string[]} A list of Kanji - */ -async function fetchKanjiFromTxt(file) { - const read = util.promisify(fs.readFile); - const data = await read(file, 'utf8'); - return data.split(''); +const txtFolder = './data/jouyou/'; +const jishoBufferFolder = './data/jisho/'; + +/* Async version of fs.readFile */ +const readFile = util.promisify(fs.readFile); + +async function fetchCharactersFromTxt(file) { + const data = await readFile(file, 'utf8'); + return [...data]; } -/** - * Reads a json file and returns the data as an object - * @param {string} file Path to file - * @returns {object} Jisho results - */ -async function fetchJishoBufferData(file) { - const read = util.promisify(fs.readFile); - const data = await read(file, 'utf8'); +async function fetchBufferedJishoResults(file) { + const data = await readFile(file, 'utf8'); return JSON.parse(data); } -/** - * Makes a delayed kanji search request in order not to overload the server. - * @param {string} kanji A character to search for - * @param {number} delay A number of milliseconds delay to the request - * @return {promise} A promise that's going to run a request after the specified delay - */ -async function delayedJishoCall(kanji, delay) { +async function makeDelayedJishoRequest(kanji, delay) { return new Promise((res, rej) => { - setTimeout(() => { - res(jisho.searchForKanji(kanji)); - }, delay); - }) + setTimeout(() => { res(jisho.searchForKanji(kanji)); }, delay); + }); } -/* Sort list of result array */ -const sortKanji = (kanjiData) => kanjiData.sort((a, b) => (a.strokeCount > b.strokeCount) ? 1 : -1); +/* Sort array of jisho results based on stroke count */ +const sortJishoResults = (jishoResult) => jishoResult.sort((a, b) => (a.strokeCount > b.strokeCount) ? 1 : -1); -/** - * Searches for kanji with a 50ms interval between each request. - * @param {string[]} kanjiArray A list of kanji to search for. - * @returns {object} JSON data containing a sorted list of search responses. - */ +/* Fetches Jisho results with a delay of 50ms between each request */ async function fetchKanjiFromJisho(kanjiArray) { - const promises = kanjiArray.map(async (kanji, i) => await delayedJishoCall(kanji, i*50)); - const data = await Promise.all(promises); - return sortKanji(data); + const delayedRequests = kanjiArray.map(async (kanji, i) => await makeDelayedJishoRequest(kanji, i*50)); + const data = await Promise.all(delayedRequests); + return sortJishoResults(data); } -exports.fetchKanjiFromTxt = fetchKanjiFromTxt; -exports.fetchJishoBufferData = fetchJishoBufferData; -exports.fetchKanjiFromJisho = fetchKanjiFromJisho; \ No newline at end of file + +async function fetchJishoDataAndWriteToBuffer(grade) { + const kanjiArray = await fetchCharactersFromTxt(`${txtFolder}${grade}.txt`); + const jishoResults = await fetchKanjiFromJisho(kanjiArray); + fs.writeFile( + `${jishoBufferFolder}${grade}.json`, + JSON.stringify(jishoResults, null, " "), + (err) => { if (err) console.error(err) } + ); + return jishoResults; +} + + /** + * Handles fetching and storing the data from Jisho + * @param {string} grade The grade of the kanji set + * @param {function} log A custom log function + * @returns {object} Jisho results + */ +async function fetchJishoResults(grade, log) { + + const bufferFileExists = fs.existsSync(`${jishoBufferFolder}${grade}.json`); + + if(bufferFileExists) { + log('Fetching Jisho data from buffer', grade) + return await fetchBufferedJishoResults(`${jishoBufferFolder}${grade}.json`); + } else { + log('Fetching data from Jisho and writing to buffer', grade) + return await fetchJishoDataAndWriteToBuffer(grade); + } +} + +exports.fetchJishoResults = fetchJishoResults; \ No newline at end of file diff --git a/jishoScrape/src/kanjiTables.js b/jishoScrape/src/kanjiTables.js index 3e0bfee..e24dc73 100644 --- a/jishoScrape/src/kanjiTables.js +++ b/jishoScrape/src/kanjiTables.js @@ -1,9 +1,9 @@ -function makeNumberRow(xLength) { - let numberRow = [...Array(xLength).keys()]; - numberRow = numberRow.map((number) => (number + 1).toString()); - numberRow = numberRow.map((number) => `{\\large ${number}}`); +function makeNumberRow(rowLength) { + let numberRow = [...Array(rowLength).keys()]; // Array containing numbers 0 to rowLength -1 + numberRow = numberRow.map((number) => (number + 1).toString()); // Correct numbers and convert to string + numberRow = numberRow.map((number) => `{\\large ${number}}`); // Encapsulate numbers in TeX code numberRow = [' ', ...numberRow]; return `${numberRow.join(' & ')} \\\\\n\\hline\n\\endhead\n`; } @@ -40,6 +40,7 @@ function makeRows(rowLength, columnLength, kanjiArray) { /** * Turns an array of kanji into a tabular for a chapter overview * @param {string[]} kanjiArray An array of kanji characters to put into the tabular + * @param {number} rowLength The length of each row * @returns {string} A tex tabular */ function chapterTabular(kanjiArray, rowLength) { diff --git a/kanjiLib.tex b/kanjiLib.tex index 94e31b3..74e98d6 100644 --- a/kanjiLib.tex +++ b/kanjiLib.tex @@ -9,12 +9,12 @@ \colorlet{meaningColor}{black} % ---------------------------------------------------------------------------- % -% JLPT Section % +% Chapter Introduction % % ---------------------------------------------------------------------------- % \setcounter{secnumdepth}{0} -\newcommand{\jlptSection}[2]{ +\newcommand{\chapterIntroduction}[2]{ % For some reason, I'm not able to use \uppercase in order to use % a single argument. In the end, I found that just using two arguments % was going to be easier than trying to override the way the LaTeX @@ -48,7 +48,7 @@ % ---------------------------------------------------------------------------- % \newcommand{\taughtIn}[1]{ - Taught in: #1 \newline + #1 \newline } \newcommand{\jlptLevel}[1]{ diff --git a/main.synctex(busy) b/main.synctex(busy) new file mode 100644 index 0000000..e69de29 diff --git a/main.tex b/main.tex index ad7e6e6..a2aba98 100644 --- a/main.tex +++ b/main.tex @@ -54,28 +54,27 @@ Kanji data is taken from \url{https://jisho.org/} - This document splits the kanjis into JLPT levels. Please note that there is no official list of kanji JLPT levels. This list is based on \href{http://www.tanos.co.uk/jlpt/skills/kanji/}{tanos.co.uk}. - \break - \jlptSection{N5}{n5} + \chapterIntroduction{Grade 1}{grade1} + \input{./data/pages/grade1.tex} - \input{./data/testing/test.tex} + \chapterIntroduction{Grade 2}{grade2} + \input{./data/pages/grade2.tex} - \jlptSection{N4}{n4} + \chapterIntroduction{Grade 3}{grade3} + \input{./data/pages/grade3.tex} - % \input{./data/pages/n4.tex} + \chapterIntroduction{Grade 4}{grade4} + \input{./data/pages/grade4.tex} - \jlptSection{N3}{n3} + \chapterIntroduction{Grade 5}{grade5} + \input{./data/pages/grade5.tex} - % \input{./data/pages/n3.tex} + \chapterIntroduction{Grade 6}{grade6} + \input{./data/pages/grade6.tex} - \jlptSection{N2}{n2} - - % \input{./data/pages/n2.tex} - - \jlptSection{N1}{n1} - - % \input{./data/pages/n1.tex} + \chapterIntroduction{Junior High}{grade7} + \input{./data/pages/grade7.tex} \end{document} \ No newline at end of file