From 8be562048a267b8473e11efa98d49816411c638c Mon Sep 17 00:00:00 2001 From: h7x4 Date: Wed, 13 Jan 2021 16:39:19 +0100 Subject: [PATCH] refactor several parts --- jishoScrape/index.js | 3 +- jishoScrape/src/chapterTabular.js | 67 +++++++++-------- jishoScrape/src/dataFetching.js | 45 +++++------- jishoScrape/src/texConversion.js | 117 +++++++++++++++--------------- kanjiLib.tex | 10 ++- main.tex | 23 ++++-- 6 files changed, 133 insertions(+), 132 deletions(-) diff --git a/jishoScrape/index.js b/jishoScrape/index.js index 4a9b47e..d896c35 100644 --- a/jishoScrape/index.js +++ b/jishoScrape/index.js @@ -26,6 +26,7 @@ async function main(grade) { let resultPage = ''; for (kanji of texData) { resultPage+=`${kanji.kanjiPageHeader} + ${kanji.label} ${kanji.kanjiMeaning ? kanji.kanjiMeaning : ''} ${kanji.kunyomi ? kanji.kunyomi : ''} ${kanji.onyomi ? kanji.onyomi : ''} @@ -56,4 +57,4 @@ async function argWrapper() { } } -argWrapper(); \ No newline at end of file +argWrapper(); diff --git a/jishoScrape/src/chapterTabular.js b/jishoScrape/src/chapterTabular.js index 09345f8..1f79652 100644 --- a/jishoScrape/src/chapterTabular.js +++ b/jishoScrape/src/chapterTabular.js @@ -1,58 +1,57 @@ -function makeNumberRow(rowLength) { - let numberRow = [...Array(rowLength).keys()]; // Array containing numbers 0 to rowLength-1 - numberRow = numberRow.map((number) => (number + 1).toString()); // Correct numbers and convert to string - numberRow = numberRow.map((number) => `{\\large ${number}}`); // Encapsulate numbers in TeX code - numberRow = [' ', ...numberRow]; + +const makeNumberRow = (length) => { + const numberRow = [ ' ', + ...[...Array(length).keys()] + .map(num => `{\\large ${num+1}}`), + ' ' ].join(' & '); return ` - ${numberRow.join(' & ')} \\\\ + ${numberRow} \\\\ \\hline \\endhead\n`; } -function kanjiRow(index, rowLength, kanjiArray) { - let result = []; - for (let rowIndex = 0; rowIndex < rowLength; rowIndex++) { - const currentIndex = index + rowIndex; - result.push(kanjiArray[currentIndex] ? kanjiArray[currentIndex] : ''); - } - return result; +const lastNonEmptyChar = (chars) => { + let index = chars.length - 1; + while (chars[index] == '') index--; + return chars[index] } -function makeRows(rowLength, columnLength, kanjiArray) { - let result = ''; - for (let columnIndex = 0; columnIndex < columnLength; columnIndex++) { - let line = new Array; - const index = columnIndex * rowLength; +const makeKanjiRow = (index, chars) => + [ `{\\large ${index}}`, + ...chars.map(chara => `\\hyperref[${chara}]{${chara}}`), + `p.\\pageref{${lastNonEmptyChar(chars)}}` ].join(' & '); - // Add the number of current character - line.push(`{\\large ${index}}`); - - // Concatenate the number with the rest of the row - line = [...line, ...kanjiRow(index, rowLength, kanjiArray)]; - - // Convert the line array into a tex row and add it to result. - result += `${line.join(' & ')} \\\\\n`; +const splitBy = (array, num) => { + let results = []; + while (array.length) { + results.push(array.splice(0, num)); } - - return result; + results[results.length-1].length = num; + results[results.length-1] = Array.from(results[results.length-1], chara => chara || ''); + return results; } +const makeRows = (length, kanjiArray) => + [ ...splitBy(kanjiArray, length) + .map((row, i) => makeKanjiRow(i*length,row)), + '' ].join(' \\\\\n'); + /** * Turns an array of kanji into a tabular for a chapter overview * @param {string[]} kanjiArray An array of kanji characters to put into the tabular * @param {number} rowLength The length of each row * @returns {string} A tex tabular */ -function chapterTabular(kanjiArray, rowLength) { - const columnLength = Math.ceil(kanjiArray.length/rowLength); +function chapterTabular(kanjiArray, length) { + // const height = Math.ceil(kanjiArray.length/length); let tabularString = ''; - tabularString += makeNumberRow(rowLength); - tabularString += makeRows(rowLength, columnLength, kanjiArray); + tabularString += makeNumberRow(length); + tabularString += makeRows(length, kanjiArray); - return `\\begin{chapterTabular}{ ${' l | ' + 'l '.repeat(rowLength)}} + return `\\begin{chapterTabular}{ ${' l | ' + 'l '.repeat(length + 1)}} ${tabularString}\\end{chapterTabular}` } -exports.chapterTabular = chapterTabular; \ No newline at end of file +exports.chapterTabular = chapterTabular; diff --git a/jishoScrape/src/dataFetching.js b/jishoScrape/src/dataFetching.js index 6bf3703..0a423d2 100644 --- a/jishoScrape/src/dataFetching.js +++ b/jishoScrape/src/dataFetching.js @@ -1,49 +1,40 @@ const fs = require('fs'); const util = require('util'); const jishoApi = require('unofficial-jisho-api'); + const jisho = new jishoApi(); const txtFolder = './data/jouyou/'; const jishoBufferFolder = './data/jisho/'; -/* Async version of fs.readFile */ -const readFile = util.promisify(fs.readFile); +const fetchCharactersFromTxt = file => [...fs.readFileSync(file, 'utf8')]; +const fetchBufferedJishoResults = file => JSON.parse(fs.readFileSync(file, 'utf8')); -async function fetchCharactersFromTxt(file) { - const data = await readFile(file, 'utf8'); - return [...data]; -} - -async function fetchBufferedJishoResults(file) { - const data = await readFile(file, 'utf8'); - return JSON.parse(data); -} - -async function makeDelayedJishoRequest(kanji, delay) { - return new Promise((res, rej) => { - setTimeout(() => { res(jisho.searchForKanji(kanji)); }, delay); - }); -} +const makeDelayedJishoRequest = (kanji, delay) => + new Promise( + (res, rej) => setTimeout(() => res(jisho.searchForKanji(kanji)), delay) + ); /* Sort array of jisho results based on stroke count */ -const sortJishoResults = (jishoResult) => jishoResult.sort((a, b) => (a.strokeCount > b.strokeCount) ? 1 : -1); +const sortJishoResults = jishoResult => jishoResult.sort((a, b) => a.strokeCount > b.strokeCount); /* Fetches Jisho results with a delay of 50ms between each request */ -async function fetchKanjiFromJisho(kanjiArray) { - const delayedRequests = kanjiArray.map(async (kanji, i) => await makeDelayedJishoRequest(kanji, i*50)); +const fetchKanjiFromJisho = async (kanjiArray) => { + const delayedRequests = kanjiArray.map((kanji, i) => makeDelayedJishoRequest(kanji, i*50)); const data = await Promise.all(delayedRequests); return sortJishoResults(data); } - async function fetchJishoDataAndWriteToBuffer(grade) { - const kanjiArray = await fetchCharactersFromTxt(`${txtFolder}${grade}.txt`); + const kanjiArray = fetchCharactersFromTxt(`${txtFolder}${grade}.txt`); const jishoResults = await fetchKanjiFromJisho(kanjiArray); - fs.writeFile( + + fs.writeFileSync( `${jishoBufferFolder}${grade}.json`, JSON.stringify(jishoResults, null, " "), (err) => { if (err) console.error(err) } ); + return jishoResults; } @@ -58,12 +49,12 @@ async function fetchJishoResults(grade, log) { const bufferFileExists = fs.existsSync(`${jishoBufferFolder}${grade}.json`); if(bufferFileExists) { - log('Fetching Jisho data from buffer', grade) - return await fetchBufferedJishoResults(`${jishoBufferFolder}${grade}.json`); + log('Fetching Jisho data from buffer', grade); + return fetchBufferedJishoResults(`${jishoBufferFolder}${grade}.json`); } else { - log('Fetching data from Jisho and writing to buffer', grade) + log('Fetching data from Jisho and writing to buffer', grade); return await fetchJishoDataAndWriteToBuffer(grade); } } -exports.fetchJishoResults = fetchJishoResults; \ No newline at end of file +exports.fetchJishoResults = fetchJishoResults; diff --git a/jishoScrape/src/texConversion.js b/jishoScrape/src/texConversion.js index fa093da..ca0c13c 100644 --- a/jishoScrape/src/texConversion.js +++ b/jishoScrape/src/texConversion.js @@ -5,77 +5,77 @@ const stylingBrackets = { const yomiConnector = '、 '; const yomiDash = '—'; -const styleText = (string) => `\\emphasize{${string}}`; +const regexes = [ + [/"|\[|\]/g, ''], // Remove all [] + [/\\/g, '\\\\'], // Escape \ + [/%/g, '\\%'], //Escape % + [/&/g, '\\&'], // Escape & + [/,/g, yomiConnector] // convert all , to yomiConnector +] -function styleCharactersBeforeDot(string) { +const styleText = string => `\\emphasize{${string}}`; + +// ab.c -> \emph{ab}.c +const styleCharactersBeforeDot = string => { const words = string.split('.'); words[0] = styleText(words[0]); return words.join(''); } -function styleEverythingExceptDash(string) { - const words = string.split(/(?<=\-)/); - if (words[0] === '-') { // ['-', 'word'] - words[0] = yomiDash; - words[1] = styleText(words[1]); - } else { // ['Word-', ''] - words[1] = yomiDash; - words[0] = words[0].slice(0, words[0].length-1); - words[0] = styleText(words[0]); +// abc- -> \emph{abc}- +const styleEverythingExceptDash = string => { + const parts = string.split(/(?<=\-)/); + if (parts[0] === '-') { // ['-', 'abc'] + parts[0] = yomiDash; + parts[1] = styleText(parts[1]); + } else { // ['abc-', ''] + parts[1] = yomiDash; + parts[0] = parts[0].slice(0, -1); + parts[0] = styleText(parts[0]); } - return words.join(''); + return parts.join(''); } -function convertKunyomi(jishoResult) { +const convertKunyomi = jishoResult => + jishoResult.kunyomi.length === 0 + ? '' + : JSON.stringify(jishoResult.kunyomi) + .replace(/"|\[|\]/g, '') + .replace(/\\/g, '\\\\') + .replace(/%/g, '\\%') + .replace(/&/g, '\\&') + .split(',') + .map(reading => { + if (reading.includes('.') && reading.includes('-')) + return `${yomiDash}${styleCharactersBeforeDot(reading.slice(1))}` + else if (reading.includes('.')) + return styleCharactersBeforeDot(reading); + else if (reading.includes('-')) + return styleEverythingExceptDash(reading); + else + return styleText(reading); + }) + .join(yomiConnector); - if (jishoResult.kunyomi.length === 0) return ''; - - const kunyomi = JSON.stringify(jishoResult.kunyomi) +const convertOnyomi = jishoResult => + JSON.stringify(jishoResult.onyomi) .replace(/"|\[|\]/g, '') .replace(/\\/g, '\\\\') .replace(/%/g, '\\%') .replace(/&/g, '\\&') - .split(','); - - for (const i in kunyomi) { - instance = kunyomi[i]; - - if (instance.includes('.') && instance.includes('-')) { - //TODO: Apply combinated logic here - } - else if (instance.includes('.')) { - kunyomi[i] = styleCharactersBeforeDot(instance); - } - else if (instance.includes('-')) { - kunyomi[i] = styleEverythingExceptDash(instance); - } - else { - kunyomi[i] = styleText(instance); - } - } - - return kunyomi.join(yomiConnector); -} - -function convertOnyomi(jishoResult) { - return JSON.stringify(jishoResult.onyomi) - .replace(/"|\[|\]/g, '') - .replace(/\\/g, '\\\\') - .replace(/%/g, '\\%') - .replace(/,/g, yomiConnector) - .replace(/&/g, '\\&'); + .split(',') + .map(styleText) + .join(yomiConnector); //TODO: Style only the words, and not the yomiConnector inbetween -} -function convertMeaning(jishoResult) { - return jishoResult.meaning +const convertMeaning = jishoResult => + jishoResult.meaning .replace(/\\/g, '\\\\') .replace(/%/g, '\\%') .replace(/&/g, '\\&'); -} -const makeFirstLetterUppercase = (string) => string.charAt(0).toUpperCase() + string.slice(1); +const makeFirstLetterUppercase = string => string.charAt(0).toUpperCase() + string.slice(1); /** * Generate TeX strings from Jisho data @@ -83,26 +83,27 @@ const makeFirstLetterUppercase = (string) => string.charAt(0).toUpperCase() + st * @param {string} grade * @returns {object} An object containg TeX strings */ -function getKanjiTexData(jishoResults, grade) { - return jishoResults.map(jishoResult => { +const getKanjiTexData = (jishoResults, grade) => { + + grade = grade.slice(0,5) + ' ' + grade.slice(5); // graden -> grade n + grade = makeFirstLetterUppercase(grade); + if (grade === 'Grade 7') grade = 'Junior High'; + + return jishoResults.map(jishoResult => { const meaning = convertMeaning(jishoResult); const kunyomi = convertKunyomi(jishoResult); const onyomi = convertOnyomi(jishoResult); - grade = grade.slice(0,5) + ' ' + grade.slice(5); - grade = makeFirstLetterUppercase(grade); - if (grade === 'Grade 7') grade = 'Junior high' - return { kanjiPageHeader: `\\kanjiPageHeader{${jishoResult.query}}{${grade}}{${jishoResult.jlptLevel}}{${jishoResult.strokeCount}}{${jishoResult.radical.symbol}}`, + label: `\\label{${jishoResult.query}}`, kanjiMeaning: meaning ? `\\kanjiMeaning{${meaning}}` : '', kunyomi: kunyomi ? `\\kunyomi{${kunyomi}}` : '', onyomi: onyomi ? `\\onyomi{${onyomi}}` : '', kanjiRow: `\\kanjiRow{${jishoResult.query}}` } - }); } -exports.getKanjiTexData = getKanjiTexData; \ No newline at end of file +exports.getKanjiTexData = getKanjiTexData; diff --git a/kanjiLib.tex b/kanjiLib.tex index f180036..50a77cf 100644 --- a/kanjiLib.tex +++ b/kanjiLib.tex @@ -12,9 +12,7 @@ % Chapter Introduction % % ---------------------------------------------------------------------------- % -\setcounter{secnumdepth}{0} - -\newcommand{\chapterIntroduction}[2]{ +\newcommand{\tocPiece}[2]{ % For some reason, I'm not able to use \uppercase in order to use % a single argument. In the end, I found that just using two arguments % was going to be easier than trying to override the way the LaTeX @@ -23,11 +21,15 @@ \begin{center} \fontsize{16}{16} + \rowcolors{1}{}{kanjiColor!20!white} \input{./data/tables/#2.tex} \end{center} +} - \localtableofcontents +\setcounter{secnumdepth}{0} +\newcommand{\chapterIntroduction}[2]{ + \tocPiece{#1}{#2} \break } diff --git a/main.tex b/main.tex index 95cc424..92ef0b3 100644 --- a/main.tex +++ b/main.tex @@ -31,7 +31,7 @@ \input{./kanjiLib.tex} -\usepackage{etoc} % For local tocs containing level based kanji list. +% \usepackage{etoc} % For local tocs containing level based kanji list. \begin{document} @@ -43,7 +43,14 @@ \break - \tableofcontents + \tocPiece{Grade 1}{grade1} + \tocPiece{Grade 2}{grade2} + \tocPiece{Grade 3}{grade3} + \tocPiece{Grade 4}{grade4} + \tocPiece{Grade 5}{grade5} + \tocPiece{Grade 6}{grade6} + \tocPiece{Junior High}{grade7} + \break \section*{Preface} @@ -60,21 +67,21 @@ \input{./data/pages/grade1.tex} \chapterIntroduction{Grade 2}{grade2} - % \input{./data/pages/grade2.tex} + \input{./data/pages/grade2.tex} \chapterIntroduction{Grade 3}{grade3} - % \input{./data/pages/grade3.tex} + \input{./data/pages/grade3.tex} \chapterIntroduction{Grade 4}{grade4} - % \input{./data/pages/grade4.tex} + \input{./data/pages/grade4.tex} \chapterIntroduction{Grade 5}{grade5} - % \input{./data/pages/grade5.tex} + \input{./data/pages/grade5.tex} \chapterIntroduction{Grade 6}{grade6} - % \input{./data/pages/grade6.tex} + \input{./data/pages/grade6.tex} \chapterIntroduction{Junior High}{grade7} - % \input{./data/pages/grade7.tex} + \input{./data/pages/grade7.tex} \end{document} \ No newline at end of file