oysteikt/kanji-sheets
oysteikt
/
kanji-sheets
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Refactor

This commit is contained in:
Oystein Kristoffer Tveit 2020-06-08 21:58:58 +02:00
parent ab85279d80
commit c82fe8cfd1
1 changed files with 34 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
jishoScrape/index.js
View File

@ -5,36 +5,22 @@ const {fetchKanjiFromTxt, fetchJishoBufferData, fetchKanjiFromJisho} = require('
const {getKanjiTexData} = require('./src/texConversion.js'); const {getKanjiTexData} = require('./src/texConversion.js');
const {kanjiTable} = require('./src/kanjiTables.js'); const {kanjiTable} = require('./src/kanjiTables.js');
function log(message, jlptLevel) {
const jlptLevelCaps = jlptLevel.toUpperCase();
console.log(`${jlptLevelCaps}: ${message}`);
}
/* Encapsulate main process in async function */ /* Encapsulate main process in async function */
async function main(jlptLevel) { async function main(jlptLevel) {
const jlptLevelCaps = jlptLevel.toUpperCase(); const jishoResults = await fetchJishoResults(jlptLevel);
const kanjiArray = jishoResults.map(result => result.query);
/* Fetch data from buffer if available. log('Generating tex pages', jlptLevel);
* Else fetch data from txt and jisho requests,
* and make buffer files
*/
if(fs.existsSync(`./data/jisho/${jlptLevel}.json`)) {
var jishoResults = await fetchJishoBufferData(`./data/jisho/${jlptLevel}.json`);
console.log(`${jlptLevelCaps}: Fetched Jisho data from buffer`);
} else {
const kanjiArray = await fetchKanjiFromTxt(`./data/txt/${jlptLevel}.txt`);
console.log(`${jlptLevelCaps}: Fetched txt`);
var jishoResults = await fetchKanjiFromJisho(kanjiArray);
console.log(`${jlptLevelCaps}: Fetched Jisho data`);
fs.writeFile(`./data/jisho/${jlptLevel}.json`, JSON.stringify(jishoResults, null, " "), (err) => {if (err) console.error(err)});
console.log(`${jlptLevelCaps}: Written Jisho data to buffer`);
}
const sortedKanjiArray = jishoResults.map(result => result.query);
const texData = getKanjiTexData(jishoResults); const texData = getKanjiTexData(jishoResults);
console.log(`${jlptLevelCaps}: Processed pages`);
const resultTable = kanjiTable(sortedKanjiArray); log('Generating chapter table page', jlptLevel);
console.log(`${jlptLevelCaps}: Processed table`); const resultTable = kanjiTable(kanjiArray);
let resultPage = ''; let resultPage = '';
for (kanji of texData) { for (kanji of texData) {
@ -50,6 +36,30 @@ async function main(jlptLevel) {
fs.writeFile(`./data/pages/${jlptLevel}.tex`, resultPage, (err) => {if (err) console.error(err)}); fs.writeFile(`./data/pages/${jlptLevel}.tex`, resultPage, (err) => {if (err) console.error(err)});
} }
/** Fetch data from buffer if available.
* Else fetch data from txt and jisho requests,
* and make buffer files
*/
async function fetchJishoResults(jlptLevel) {
const bufferFileExists = fs.existsSync(`./data/jisho/${jlptLevel}.json`);
if(bufferFileExists) {
log('Fetching Jisho data from buffer', jlptLevel)
return await fetchJishoBufferData(`./data/jisho/${jlptLevel}.json`);
} else {
log('Fetching data from Jisho and writing to buffer', jlptLevel)
return await fetchJishoDataAndWriteToBuffer(jlptLevel);
}
}
async function fetchJishoDataAndWriteToBuffer(jlptLevel) {
const kanjiArray = await fetchKanjiFromTxt(`./data/txt/${jlptLevel}.txt`);
const jishoResults = await fetchKanjiFromJisho(kanjiArray);
fs.writeFile(`./data/jisho/${jlptLevel}.json`, JSON.stringify(jishoResults, null, " "), (err) => {if (err) console.error(err)});
return jishoResults;
}
/* Handle args */ /* Handle args */
async function argWrapper() { async function argWrapper() {