Split into files

This commit is contained in:
Oystein Kristoffer Tveit 2020-06-22 15:18:48 +02:00
parent 39c2eed084
commit 86b8ccfdbb
8 changed files with 488 additions and 779 deletions

3
lib/src/baseURI.dart Normal file
View File

@ -0,0 +1,3 @@
const String JISHO_API = 'https://jisho.org/api/v1/search/words';
const String SCRAPE_BASE_URI = 'https://jisho.org/search/';
const String STROKE_ORDER_DIAGRAM_BASE_URI = 'https://classic.jisho.org/static/images/stroke_diagrams/';

108
lib/src/exampleSearch.dart Normal file
View File

@ -0,0 +1,108 @@
import './baseURI.dart';
import './objects.dart';
import 'package:html/parser.dart';
import 'package:html/dom.dart';
final RegExp kanjiRegex = RegExp(r'[\u4e00-\u9faf\u3400-\u4dbf]');
String uriForExampleSearch(String phrase) {
return '${SCRAPE_BASE_URI}${Uri.encodeComponent(phrase)}%23sentences';
}
/* TODO: This is the wrong approach.
* Symbols such as are missing in mid sentence
* Maybe also JP fullwidth numbers?
*/
String getEndSymbolsOfExampleSentence(Element ul) {
final endSymbols = RegExp(r'<\/li>([^<>]+)$');
return endSymbols.firstMatch(ul.innerHtml).group(1);
}
ExampleResultData getKanjiAndKana(Element div) {
final ul = div.querySelector('ul');
final contents = ul.children;
var kanji = '';
var kana = '';
for (var i = 0; i < contents.length; i += 1) {
final content = contents[i];
if (content.localName == 'li') {
final li = content;
final furigana = li.querySelector('.furigana')?.text;
final unlifted = li.querySelector('.unlinked')?.text;
if (furigana != null) {
kanji += unlifted;
kana += furigana;
final kanaEnding = [];
for (var j = unlifted.length - 1; j > 0; j -= 1) {
final char = unlifted[j];
if (!kanjiRegex.hasMatch(char)) {
kanaEnding.add(char);
} else {
break;
}
}
kana += kanaEnding.reversed.join('');
} else {
kanji += unlifted;
kana += unlifted;
}
} else {
final text = content.text.trim();
if (text != null) {
kanji += text;
kana += text;
}
}
}
final endSymbols = getEndSymbolsOfExampleSentence(ul).trim();
kanji+= endSymbols;
kana += endSymbols;
return ExampleResultData(
kanji: kanji,
kana: kana,
);
}
List<ExampleSentencePiece> getPieces(Element sentenceElement) {
final pieceElements = sentenceElement.querySelectorAll('li.clearfix');
final List<ExampleSentencePiece> pieces = [];
for (var pieceIndex = 0; pieceIndex < pieceElements.length; pieceIndex += 1) {
final pieceElement = pieceElements[pieceIndex];
pieces.add(ExampleSentencePiece(
lifted: pieceElement.querySelector('.furigana')?.text,
unlifted: pieceElement.querySelector('.unlinked')?.text,
));
}
return pieces;
}
ExampleResultData parseExampleDiv(Element div) {
final result = getKanjiAndKana(div);
result.english = div.querySelector('.english').text;
result.pieces = getPieces(div) ?? [];
return result;
}
ExampleResults parseExamplePageData(String pageHtml, String phrase) {
final document = parse(pageHtml);
final divs = document.querySelectorAll('.sentence_content');
final results = divs.map((div) => parseExampleDiv(div)).toList();
return ExampleResults(
query: phrase,
found: results.isNotEmpty,
results: results ?? [],
uri: uriForExampleSearch(phrase),
phrase: phrase,
);
}

214
lib/src/kanjiSearch.dart Normal file
View File

@ -0,0 +1,214 @@
import './baseURI.dart';
import './objects.dart';
import 'package:html_unescape/html_unescape.dart' as html_entities;
final htmlUnescape = html_entities.HtmlUnescape();
const String ONYOMI_LOCATOR_SYMBOL = 'On';
const String KUNYOMI_LOCATOR_SYMBOL = 'Kun';
String removeNewlines(String str) {
return str.replaceAll(RegExp(r'(?:\r|\n)') , '').trim();
}
String uriForKanjiSearch(String kanji) {
return '${SCRAPE_BASE_URI}${Uri.encodeComponent(kanji)}%23kanji';
}
String getUriForStrokeOrderDiagram(String kanji) {
return '${STROKE_ORDER_DIAGRAM_BASE_URI}${kanji.codeUnitAt(0)}_frames.png';
}
bool containsKanjiGlyph(String pageHtml, String kanji) {
final kanjiGlyphToken = '<h1 class="character" data-area-name="print" lang="ja">${kanji}</h1>';
return pageHtml.contains(kanjiGlyphToken);
}
String getStringBetweenIndicies(String data, int startIndex, int endIndex) {
final result = data.substring(startIndex, endIndex);
return removeNewlines(result).trim();
}
String getStringBetweenStrings(String data, String startString, String endString) {
final regex = RegExp('${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}', dotAll: true);
final match = regex.allMatches(data).toList();
return match.isNotEmpty ? match[0].group(1).toString() : null;
}
int getIntBetweenStrings(String pageHtml, String startString, String endString) {
final stringBetweenStrings = getStringBetweenStrings(pageHtml, startString, endString);
return int.parse(stringBetweenStrings);
}
List<String> getAllGlobalGroupMatches(String str, RegExp regex) {
var regexResults = regex.allMatches(str).toList();
List<String> results = [];
for (var match in regexResults) {
results.add(match.group(1));
}
return results;
}
List<String> parseAnchorsToArray(String str) {
final regex = RegExp(r'<a href=".*?">(.*?)<\/a>');
return getAllGlobalGroupMatches(str, regex);
}
List<String> getYomi(String pageHtml, String yomiLocatorSymbol) {
final yomiSection = getStringBetweenStrings(pageHtml, '<dt>${yomiLocatorSymbol}:</dt>', '</dl>');
return parseAnchorsToArray(yomiSection ?? '');
}
List<String> getKunyomi(String pageHtml) {
return getYomi(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
}
List<String> getOnyomi(String pageHtml) {
return getYomi(pageHtml, ONYOMI_LOCATOR_SYMBOL);
}
List<YomiExample> getYomiExamples(String pageHtml, String yomiLocatorSymbol) {
final locatorString = '<h2>${yomiLocatorSymbol} reading compounds</h2>';
final exampleSection = getStringBetweenStrings(pageHtml, locatorString, '</ul>');
if (exampleSection==null) {
return null;
}
final regex = RegExp(r'<li>(.*?)<\/li>', dotAll: true);
final regexResults = getAllGlobalGroupMatches(exampleSection, regex).map((s) => s.trim());
final examples = regexResults.map((regexResult) {
final examplesLines = regexResult.split('\n').map((s) => s.trim()).toList();
return YomiExample(
example: examplesLines[0],
reading: examplesLines[1].replaceAll('', '').replaceAll('', ''),
meaning: htmlUnescape.convert(examplesLines[2]),
);
});
return examples.toList();
}
List<YomiExample> getOnyomiExamples(String pageHtml) {
return getYomiExamples(pageHtml, ONYOMI_LOCATOR_SYMBOL);
}
List<YomiExample> getKunyomiExamples(String pageHtml) {
return getYomiExamples(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
}
Radical getRadical(String pageHtml) {
const radicalMeaningStartString = '<span class="radical_meaning">';
const radicalMeaningEndString = '</span>';
var radicalMeaning = getStringBetweenStrings(
pageHtml,
radicalMeaningStartString,
radicalMeaningEndString,
).trim();
if (radicalMeaning!=null) {
final radicalMeaningStartIndex = pageHtml.indexOf(radicalMeaningStartString);
final radicalMeaningEndIndex = pageHtml.indexOf(
radicalMeaningEndString,
radicalMeaningStartIndex,
);
final radicalSymbolStartIndex = radicalMeaningEndIndex + radicalMeaningEndString.length;
const radicalSymbolEndString = '</span>';
final radicalSymbolEndIndex = pageHtml.indexOf(radicalSymbolEndString, radicalSymbolStartIndex);
final radicalSymbolsString = getStringBetweenIndicies(
pageHtml,
radicalSymbolStartIndex,
radicalSymbolEndIndex,
);
if (radicalSymbolsString.length > 1) {
final radicalForms = radicalSymbolsString
.substring(1)
.replaceAll('(', '')
.replaceAll(')', '')
.trim()
.split(', ');
return Radical(
symbol: radicalSymbolsString[0],
forms: radicalForms ?? [],
meaning: radicalMeaning
);
}
return Radical (
symbol: radicalSymbolsString,
meaning: radicalMeaning
);
}
return null;
}
List<String> getParts(String pageHtml) {
const partsSectionStartString = '<dt>Parts:</dt>';
const partsSectionEndString = '</dl>';
final partsSection = getStringBetweenStrings(
pageHtml,
partsSectionStartString,
partsSectionEndString,
);
var result = parseAnchorsToArray(partsSection);
result.sort();
return (result);
}
String getSvgUri(String pageHtml) {
var svgRegex = RegExp('\/\/.*?.cloudfront.net\/.*?.svg');
final regexResult = svgRegex.firstMatch(pageHtml).group(0).toString();
return regexResult.isNotEmpty ? 'https:${regexResult}' : null;
}
String getGifUri(String kanji) {
final unicodeString = kanji.codeUnitAt(0).toRadixString(16);
final fileName = '${unicodeString}.gif';
final animationUri = 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/${fileName}';
return animationUri;
}
int getNewspaperFrequencyRank(String pageHtml) {
final frequencySection = getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>');
return (frequencySection != null) ? int.parse(getStringBetweenStrings(frequencySection, '<strong>', '</strong>')) : null;
}
KanjiResult parseKanjiPageData(String pageHtml, String kanji) {
final result = KanjiResult();
result.query = kanji;
result.found = containsKanjiGlyph(pageHtml, kanji);
if (result.found==false) {
return result;
}
result.taughtIn = getStringBetweenStrings(pageHtml, 'taught in <strong>', '</strong>');
result.jlptLevel = getStringBetweenStrings(pageHtml, 'JLPT level <strong>', '</strong>');
result.newspaperFrequencyRank = getNewspaperFrequencyRank(pageHtml);
result.strokeCount = getIntBetweenStrings(pageHtml, '<strong>', '</strong> strokes');
result.meaning = htmlUnescape.convert(removeNewlines(getStringBetweenStrings(pageHtml, '<div class="kanji-details__main-meanings">', '</div>')).trim());
result.kunyomi = getKunyomi(pageHtml) ?? [];
result.onyomi = getOnyomi(pageHtml) ?? [];
result.onyomiExamples = getOnyomiExamples(pageHtml) ?? [];
result.kunyomiExamples = getKunyomiExamples(pageHtml) ?? [];
result.radical = getRadical(pageHtml);
result.parts = getParts(pageHtml) ?? [];
result.strokeOrderDiagramUri = getUriForStrokeOrderDiagram(kanji);
result.strokeOrderSvgUri = getSvgUri(pageHtml);
result.strokeOrderGifUri = getGifUri(kanji);
result.uri = uriForKanjiSearch(kanji);
return result;
}

154
lib/src/phraseScrape.dart Normal file
View File

@ -0,0 +1,154 @@
import './objects.dart';
import './exampleSearch.dart';
import 'package:html/parser.dart';
import 'package:html/dom.dart';
List<String> getTags(Document document) {
final List<String> tags = [];
final tagElements = document.querySelectorAll('.concept_light-tag');
for (var i = 0; i < tagElements.length; i += 1) {
final tagText = tagElements[i].text;
tags.add(tagText);
}
return tags;
}
List<String> getMostRecentWordTypes(Element child) {
return child.text.split(',').map((s) => s.trim().toLowerCase()).toList();
}
List<KanjiKanaPair> getOtherForms(Element child) {
return child.text.split('')
.map((s) => s.replaceAll('', '').replaceAll('', '').split(' '))
.map((a) => (KanjiKanaPair( kanji: a[0], kana: (a.length == 2) ? a[1] : null ))).toList();
}
List<String> getNotes(Element child) => child.text.split('\n');
String getMeaning(Element child) => child.querySelector('.meaning-meaning').text;
String getMeaningAbstract(Element child) {
final meaningAbstract = child.querySelector('.meaning-abstract');
if (meaningAbstract == null) return null;
for (var element in meaningAbstract.querySelectorAll('a')) {
element.remove();
}
return child.querySelector('.meaning-abstract')?.text;
}
List<String> getSupplemental(Element child) {
final supplemental = child.querySelector('.supplemental_info');
if (supplemental == null) return [];
return supplemental.text.split(',').map((s) => s.trim()).toList();
}
List<String> getSeeAlsoTerms(List<String> supplemental) {
if (supplemental == null) return [];
final List<String> seeAlsoTerms = [];
for (var i = supplemental.length - 1; i >= 0; i -= 1) {
final supplementalEntry = supplemental[i];
if (supplementalEntry.startsWith('See also')) {
seeAlsoTerms.add(supplementalEntry.replaceAll('See also ', ''));
supplemental.removeAt(i);
}
}
return seeAlsoTerms;
}
List<PhraseScrapeSentence> getSentences(Element child) {
final sentenceElements = child.querySelector('.sentences')?.querySelectorAll('.sentence');
if (sentenceElements == null) return [];
final List<PhraseScrapeSentence> sentences = [];
for (var sentenceIndex = 0; sentenceIndex < (sentenceElements?.length ?? 0); sentenceIndex += 1) {
final sentenceElement = sentenceElements[sentenceIndex];
final english = sentenceElement.querySelector('.english').text;
final pieces = getPieces(sentenceElement);
sentenceElement.querySelector('.english').remove();
for (var element in sentenceElement.children[0].children) {
element.querySelector('.furigana')?.remove();
}
final japanese = sentenceElement.text;
sentences.add(
PhraseScrapeSentence(
english: english,
japanese: japanese,
pieces: pieces ?? []
)
);
}
return sentences;
}
PhrasePageScrapeResult getMeaningsOtherFormsAndNotes(Document document) {
final returnValues = PhrasePageScrapeResult( otherForms: [], notes: [] );
final meaningsWrapper = document.querySelector('.meanings-wrapper');
if (meaningsWrapper == null) return PhrasePageScrapeResult(found: false);
returnValues.found = true;
final meaningsChildren = meaningsWrapper.children;
final List<PhraseScrapeMeaning> meanings = [];
var mostRecentWordTypes = [];
for (var meaningIndex = 0; meaningIndex < meaningsChildren.length; meaningIndex += 1) {
final child = meaningsChildren[meaningIndex];
if (child.className.contains('meaning-tags')) {
mostRecentWordTypes = getMostRecentWordTypes(child);
} else if (mostRecentWordTypes[0] == 'other forms') {
returnValues.otherForms = getOtherForms(child);
} else if (mostRecentWordTypes[0] == 'notes') {
returnValues.notes = getNotes(child);
} else {
final meaning = getMeaning(child);
final meaningAbstract = getMeaningAbstract(child);
final supplemental = getSupplemental(child);
final seeAlsoTerms = getSeeAlsoTerms(supplemental);
final sentences = getSentences(child);
meanings.add(PhraseScrapeMeaning(
seeAlsoTerms: seeAlsoTerms ?? [],
sentences: sentences ?? [],
definition: meaning,
supplemental: supplemental ?? [],
definitionAbstract: meaningAbstract,
tags: mostRecentWordTypes ?? [],
));
}
}
returnValues.meanings = meanings;
return returnValues;
}
String uriForPhraseScrape(String searchTerm) {
return 'https://jisho.org/word/${Uri.encodeComponent(searchTerm)}';
}
PhrasePageScrapeResult parsePhrasePageData(String pageHtml, String query) {
final document = parse(pageHtml);
final result = getMeaningsOtherFormsAndNotes(document);
result.query = query;
if (!result.found) return result;
result.uri = uriForPhraseScrape(query);
result.tags = getTags(document);
return result;
}

View File

@ -0,0 +1,5 @@
import './baseURI.dart';
String uriForPhraseSearch(String phrase) {
return '${JISHO_API}?keyword=${Uri.encodeComponent(phrase)}';
}

View File

@ -1,489 +1,12 @@
import 'package:unofficial_jisho_api/src/objects.dart'; import 'package:unofficial_jisho_api/src/objects.dart';
import 'package:http/http.dart' as http; import 'package:http/http.dart' as http;
import 'package:html_unescape/html_unescape.dart' as html_entities;
import 'dart:convert'; import 'dart:convert';
import 'package:html/parser.dart';
import 'package:html/dom.dart';
final htmlUnescape = html_entities.HtmlUnescape(); import './phraseSearch.dart';
import './kanjiSearch.dart';
import './exampleSearch.dart';
import './phraseScrape.dart';
const String JISHO_API = 'https://jisho.org/api/v1/search/words';
const String SCRAPE_BASE_URI = 'https://jisho.org/search/';
const String STROKE_ORDER_DIAGRAM_BASE_URI = 'https://classic.jisho.org/static/images/stroke_diagrams/';
/* KANJI SEARCH FUNCTIONS START */
const String ONYOMI_LOCATOR_SYMBOL = 'On';
const KUNYOMI_LOCATOR_SYMBOL = 'Kun';
String removeNewlines(String str) {
return str.replaceAll(RegExp(r'(?:\r|\n)') , '').trim();
}
String uriForKanjiSearch(String kanji) {
return '${SCRAPE_BASE_URI}${Uri.encodeComponent(kanji)}%23kanji';
}
String getUriForStrokeOrderDiagram(String kanji) {
return '${STROKE_ORDER_DIAGRAM_BASE_URI}${kanji.codeUnitAt(0)}_frames.png';
}
String uriForPhraseSearch(String phrase) {
return '${JISHO_API}?keyword=${Uri.encodeComponent(phrase)}';
}
bool containsKanjiGlyph(String pageHtml, String kanji) {
final kanjiGlyphToken = '<h1 class="character" data-area-name="print" lang="ja">${kanji}</h1>';
return pageHtml.contains(kanjiGlyphToken);
}
String getStringBetweenIndicies(String data, int startIndex, int endIndex) {
final result = data.substring(startIndex, endIndex);
return removeNewlines(result).trim();
}
String getStringBetweenStrings(String data, String startString, String endString) {
final regex = RegExp('${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}', dotAll: true);
final match = regex.allMatches(data).toList();
return match.isNotEmpty ? match[0].group(1).toString() : null;
}
int getIntBetweenStrings(String pageHtml, String startString, String endString) {
final stringBetweenStrings = getStringBetweenStrings(pageHtml, startString, endString);
return int.parse(stringBetweenStrings);
}
List<String> getAllGlobalGroupMatches(String str, RegExp regex) {
var regexResults = regex.allMatches(str).toList();
List<String> results = [];
for (var match in regexResults) {
results.add(match.group(1));
}
return results;
}
List<String> parseAnchorsToArray(String str) {
final regex = RegExp(r'<a href=".*?">(.*?)<\/a>');
return getAllGlobalGroupMatches(str, regex);
}
List<String> getYomi(String pageHtml, String yomiLocatorSymbol) {
final yomiSection = getStringBetweenStrings(pageHtml, '<dt>${yomiLocatorSymbol}:</dt>', '</dl>');
return parseAnchorsToArray(yomiSection ?? '');
}
List<String> getKunyomi(String pageHtml) {
return getYomi(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
}
List<String> getOnyomi(String pageHtml) {
return getYomi(pageHtml, ONYOMI_LOCATOR_SYMBOL);
}
List<YomiExample> getYomiExamples(String pageHtml, String yomiLocatorSymbol) {
final locatorString = '<h2>${yomiLocatorSymbol} reading compounds</h2>';
final exampleSection = getStringBetweenStrings(pageHtml, locatorString, '</ul>');
if (exampleSection==null) {
return null;
}
final regex = RegExp(r'<li>(.*?)<\/li>', dotAll: true);
final regexResults = getAllGlobalGroupMatches(exampleSection, regex).map((s) => s.trim());
final examples = regexResults.map((regexResult) {
final examplesLines = regexResult.split('\n').map((s) => s.trim()).toList();
return YomiExample(
example: examplesLines[0],
reading: examplesLines[1].replaceAll('', '').replaceAll('', ''),
meaning: htmlUnescape.convert(examplesLines[2]),
);
});
return examples.toList();
}
List<YomiExample> getOnyomiExamples(String pageHtml) {
return getYomiExamples(pageHtml, ONYOMI_LOCATOR_SYMBOL);
}
List<YomiExample> getKunyomiExamples(String pageHtml) {
return getYomiExamples(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
}
Radical getRadical(String pageHtml) {
const radicalMeaningStartString = '<span class="radical_meaning">';
const radicalMeaningEndString = '</span>';
var radicalMeaning = getStringBetweenStrings(
pageHtml,
radicalMeaningStartString,
radicalMeaningEndString,
).trim();
if (radicalMeaning!=null) {
final radicalMeaningStartIndex = pageHtml.indexOf(radicalMeaningStartString);
final radicalMeaningEndIndex = pageHtml.indexOf(
radicalMeaningEndString,
radicalMeaningStartIndex,
);
final radicalSymbolStartIndex = radicalMeaningEndIndex + radicalMeaningEndString.length;
const radicalSymbolEndString = '</span>';
final radicalSymbolEndIndex = pageHtml.indexOf(radicalSymbolEndString, radicalSymbolStartIndex);
final radicalSymbolsString = getStringBetweenIndicies(
pageHtml,
radicalSymbolStartIndex,
radicalSymbolEndIndex,
);
if (radicalSymbolsString.length > 1) {
final radicalForms = radicalSymbolsString
.substring(1)
.replaceAll('(', '')
.replaceAll(')', '')
.trim()
.split(', ');
return Radical(
symbol: radicalSymbolsString[0],
forms: radicalForms ?? [],
meaning: radicalMeaning
);
}
return Radical (
symbol: radicalSymbolsString,
meaning: radicalMeaning
);
}
return null;
}
List<String> getParts(String pageHtml) {
const partsSectionStartString = '<dt>Parts:</dt>';
const partsSectionEndString = '</dl>';
final partsSection = getStringBetweenStrings(
pageHtml,
partsSectionStartString,
partsSectionEndString,
);
var result = parseAnchorsToArray(partsSection);
result.sort();
return (result);
}
String getSvgUri(String pageHtml) {
var svgRegex = RegExp('\/\/.*?.cloudfront.net\/.*?.svg');
final regexResult = svgRegex.firstMatch(pageHtml).group(0).toString();
return regexResult.isNotEmpty ? 'https:${regexResult}' : null;
}
String getGifUri(String kanji) {
final unicodeString = kanji.codeUnitAt(0).toRadixString(16);
final fileName = '${unicodeString}.gif';
final animationUri = 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/${fileName}';
return animationUri;
}
int getNewspaperFrequencyRank(String pageHtml) {
final frequencySection = getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>');
return (frequencySection != null) ? int.parse(getStringBetweenStrings(frequencySection, '<strong>', '</strong>')) : null;
}
KanjiResult parseKanjiPageData(String pageHtml, String kanji) {
final result = KanjiResult();
result.query = kanji;
result.found = containsKanjiGlyph(pageHtml, kanji);
if (result.found==false) {
return result;
}
result.taughtIn = getStringBetweenStrings(pageHtml, 'taught in <strong>', '</strong>');
result.jlptLevel = getStringBetweenStrings(pageHtml, 'JLPT level <strong>', '</strong>');
result.newspaperFrequencyRank = getNewspaperFrequencyRank(pageHtml);
result.strokeCount = getIntBetweenStrings(pageHtml, '<strong>', '</strong> strokes');
result.meaning = htmlUnescape.convert(removeNewlines(getStringBetweenStrings(pageHtml, '<div class="kanji-details__main-meanings">', '</div>')).trim());
result.kunyomi = getKunyomi(pageHtml) ?? [];
result.onyomi = getOnyomi(pageHtml) ?? [];
result.onyomiExamples = getOnyomiExamples(pageHtml) ?? [];
result.kunyomiExamples = getKunyomiExamples(pageHtml) ?? [];
result.radical = getRadical(pageHtml);
result.parts = getParts(pageHtml) ?? [];
result.strokeOrderDiagramUri = getUriForStrokeOrderDiagram(kanji);
result.strokeOrderSvgUri = getSvgUri(pageHtml);
result.strokeOrderGifUri = getGifUri(kanji);
result.uri = uriForKanjiSearch(kanji);
return result;
}
/* KANJI SEARCH FUNCTIONS END */
/* EXAMPLE SEARCH FUNCTIONS START */
final RegExp kanjiRegex = RegExp(r'[\u4e00-\u9faf\u3400-\u4dbf]');
String uriForExampleSearch(String phrase) {
return '${SCRAPE_BASE_URI}${Uri.encodeComponent(phrase)}%23sentences';
}
/* TODO: This is the wrong approach.
* Symbols such as are missing in mid sentence
* Maybe also JP fullwidth numbers?
*/
String getEndSymbolsOfExampleSentence(Element ul) {
final endSymbols = RegExp(r'<\/li>([^<>]+)$');
return endSymbols.firstMatch(ul.innerHtml).group(1);
}
ExampleResultData getKanjiAndKana(Element div) {
final ul = div.querySelector('ul');
final contents = ul.children;
var kanji = '';
var kana = '';
for (var i = 0; i < contents.length; i += 1) {
final content = contents[i];
if (content.localName == 'li') {
final li = content;
final furigana = li.querySelector('.furigana')?.text;
final unlifted = li.querySelector('.unlinked')?.text;
if (furigana != null) {
kanji += unlifted;
kana += furigana;
final kanaEnding = [];
for (var j = unlifted.length - 1; j > 0; j -= 1) {
final char = unlifted[j];
if (!kanjiRegex.hasMatch(char)) {
kanaEnding.add(char);
} else {
break;
}
}
kana += kanaEnding.reversed.join('');
} else {
kanji += unlifted;
kana += unlifted;
}
} else {
final text = content.text.trim();
if (text != null) {
kanji += text;
kana += text;
}
}
}
final endSymbols = getEndSymbolsOfExampleSentence(ul).trim();
kanji+= endSymbols;
kana += endSymbols;
return ExampleResultData(
kanji: kanji,
kana: kana,
);
}
List<ExampleSentencePiece> getPieces(Element sentenceElement) {
final pieceElements = sentenceElement.querySelectorAll('li.clearfix');
final List<ExampleSentencePiece> pieces = [];
for (var pieceIndex = 0; pieceIndex < pieceElements.length; pieceIndex += 1) {
final pieceElement = pieceElements[pieceIndex];
pieces.add(ExampleSentencePiece(
lifted: pieceElement.querySelector('.furigana')?.text,
unlifted: pieceElement.querySelector('.unlinked')?.text,
));
}
return pieces;
}
ExampleResultData parseExampleDiv(Element div) {
final result = getKanjiAndKana(div);
result.english = div.querySelector('.english').text;
result.pieces = getPieces(div) ?? [];
return result;
}
ExampleResults parseExamplePageData(String pageHtml, String phrase) {
final document = parse(pageHtml);
final divs = document.querySelectorAll('.sentence_content');
final results = divs.map((div) => parseExampleDiv(div)).toList();
return ExampleResults(
query: phrase,
found: results.isNotEmpty,
results: results ?? [],
uri: uriForExampleSearch(phrase),
phrase: phrase,
);
}
/* EXAMPLE SEARCH FUNCTIONS END */
/* PHRASE SCRAPE FUNCTIONS START */
List<String> getTags(Document document) {
final List<String> tags = [];
final tagElements = document.querySelectorAll('.concept_light-tag');
for (var i = 0; i < tagElements.length; i += 1) {
final tagText = tagElements[i].text;
tags.add(tagText);
}
return tags;
}
List<String> getMostRecentWordTypes(Element child) {
return child.text.split(',').map((s) => s.trim().toLowerCase()).toList();
}
List<KanjiKanaPair> getOtherForms(Element child) {
return child.text.split('')
.map((s) => s.replaceAll('', '').replaceAll('', '').split(' '))
.map((a) => (KanjiKanaPair( kanji: a[0], kana: (a.length == 2) ? a[1] : null ))).toList();
}
List<String> getNotes(Element child) => child.text.split('\n');
String getMeaning(Element child) => child.querySelector('.meaning-meaning').text;
String getMeaningAbstract(Element child) {
final meaningAbstract = child.querySelector('.meaning-abstract');
if (meaningAbstract == null) return null;
for (var element in meaningAbstract.querySelectorAll('a')) {
element.remove();
}
return child.querySelector('.meaning-abstract')?.text;
}
List<String> getSupplemental(Element child) {
final supplemental = child.querySelector('.supplemental_info');
if (supplemental == null) return [];
return supplemental.text.split(',').map((s) => s.trim()).toList();
}
List<String> getSeeAlsoTerms(List<String> supplemental) {
if (supplemental == null) return [];
final List<String> seeAlsoTerms = [];
for (var i = supplemental.length - 1; i >= 0; i -= 1) {
final supplementalEntry = supplemental[i];
if (supplementalEntry.startsWith('See also')) {
seeAlsoTerms.add(supplementalEntry.replaceAll('See also ', ''));
supplemental.removeAt(i);
}
}
return seeAlsoTerms;
}
List<PhraseScrapeSentence> getSentences(Element child) {
final sentenceElements = child.querySelector('.sentences')?.querySelectorAll('.sentence');
if (sentenceElements == null) return [];
final List<PhraseScrapeSentence> sentences = [];
for (var sentenceIndex = 0; sentenceIndex < (sentenceElements?.length ?? 0); sentenceIndex += 1) {
final sentenceElement = sentenceElements[sentenceIndex];
final english = sentenceElement.querySelector('.english').text;
final pieces = getPieces(sentenceElement);
sentenceElement.querySelector('.english').remove();
for (var element in sentenceElement.children[0].children) {
element.querySelector('.furigana')?.remove();
}
final japanese = sentenceElement.text;
sentences.add(
PhraseScrapeSentence(
english: english,
japanese: japanese,
pieces: pieces ?? []
)
);
}
return sentences;
}
PhrasePageScrapeResult getMeaningsOtherFormsAndNotes(Document document) {
final returnValues = PhrasePageScrapeResult( otherForms: [], notes: [] );
final meaningsWrapper = document.querySelector('.meanings-wrapper');
if (meaningsWrapper == null) return PhrasePageScrapeResult(found: false);
returnValues.found = true;
final meaningsChildren = meaningsWrapper.children;
final List<PhraseScrapeMeaning> meanings = [];
var mostRecentWordTypes = [];
for (var meaningIndex = 0; meaningIndex < meaningsChildren.length; meaningIndex += 1) {
final child = meaningsChildren[meaningIndex];
if (child.className.contains('meaning-tags')) {
mostRecentWordTypes = getMostRecentWordTypes(child);
} else if (mostRecentWordTypes[0] == 'other forms') {
returnValues.otherForms = getOtherForms(child);
} else if (mostRecentWordTypes[0] == 'notes') {
returnValues.notes = getNotes(child);
} else {
final meaning = getMeaning(child);
final meaningAbstract = getMeaningAbstract(child);
final supplemental = getSupplemental(child);
final seeAlsoTerms = getSeeAlsoTerms(supplemental);
final sentences = getSentences(child);
meanings.add(PhraseScrapeMeaning(
seeAlsoTerms: seeAlsoTerms ?? [],
sentences: sentences ?? [],
definition: meaning,
supplemental: supplemental ?? [],
definitionAbstract: meaningAbstract,
tags: mostRecentWordTypes ?? [],
));
}
}
returnValues.meanings = meanings;
return returnValues;
}
String uriForPhraseScrape(String searchTerm) {
return 'https://jisho.org/word/${Uri.encodeComponent(searchTerm)}';
}
PhrasePageScrapeResult parsePhrasePageData(String pageHtml, String query) {
final document = parse(pageHtml);
final result = getMeaningsOtherFormsAndNotes(document);
result.query = query;
if (!result.found) return result;
result.uri = uriForPhraseScrape(query);
result.tags = getTags(document);
return result;
}
class JishoApi { class JishoApi {

View File

@ -1,293 +0,0 @@
import 'package:unofficial_jisho_api/src/objects.dart';
import 'package:unofficial_jisho_api/unofficial_jisho_api.dart';
import 'package:test/test.dart';
import 'dart:convert';
import 'package:http/http.dart' as http;
void test_local_functions() async {
/* KANJI SEARCH FUNCTION TESTS START */
test('removeNewLines', () {
final result = removeNewlines('Line \nwith\r\n Newlines and spaces\n');
expect(result, 'Line with Newlines and spaces');
});
test('uriForKanjiSearch', () {
final result = uriForKanjiSearch('');
expect(result, 'https://jisho.org/search/%E6%99%82%23kanji');
});
test('getUriForStrokeOrderDiagram', () {
final result = getUriForStrokeOrderDiagram('');
expect(result, 'https://classic.jisho.org/static/images/stroke_diagrams/26178_frames.png');
});
test('uriForPhraseSearch', () {
final result = uriForPhraseSearch('時間');
expect(result, 'https://jisho.org/api/v1/search/words?keyword=%E6%99%82%E9%96%93');
});
final kanjiPage = (await http.get('https://jisho.org/search/%E6%99%82%23kanji')).body;
test('containsKanjiGlyph', () {
final result = containsKanjiGlyph(kanjiPage, '');
expect(result, true);
});
test('getStringBetweenIndicies', () {
final result = getStringBetweenIndicies('String\n\rwith\nNewlines', 3, 9);
expect(result, 'ingw');
});
test('getStringBetweenStrings', () {
const data = 'STArT I want this string END';
final result = getStringBetweenStrings(data, 'STArT', 'END');
expect(result, ' I want this string ');
});
test('getIntBetweenStrings', () {
final result = getIntBetweenStrings(kanjiPage, '<strong>', '</strong> strokes');
expect(result, 10);
});
test('getAllGlobalGroupMatches', () {
});
test('parseAnchorsToArray', () {
const htmlCode =
'''
<div class="test">
<p>
<a href="https://test.test">Hello</a>
</p>
<a href="//xyz">Hi</a>
<span>
<p>
<a href="">How are you doing</a>
</p>
</span>
</div>
''';
final result = parseAnchorsToArray(htmlCode);
expect(result, [
'Hello', 'Hi', 'How are you doing']);
});
test('getYomi', () {
final result = getYomi(kanjiPage, 'On');
expect(result, ['']);
});
test('getKunyomi', () {
final result = getKunyomi(kanjiPage);
expect(result, ['とき', '-どき']);
});
test('getOnyomi', () {
final result = getOnyomi(kanjiPage);
expect(result, ['']);
});
test('getYomiExamples', () {
final result = getYomiExamples(kanjiPage, 'On');
expect(
json.encode(result),
json.encode([
YomiExample(
example: '',
reading: '',
meaning: '''hour, o'clock, (specified) time, when ..., during ...'''
),
YomiExample(
example: '時価',
reading: 'ジカ',
meaning: 'current value, price, market value'
),
YomiExample(
example: '零時',
reading: 'レイジ',
meaning: '''twelve o'clock, midnight, noon'''
),
YomiExample(
example: '平時',
reading: 'ヘイジ',
meaning: 'peacetime, time of peace, ordinary times, normal times'
),
])
);
});
test('getOnyomiExamples', () {
final result = getOnyomiExamples(kanjiPage);
expect(
json.encode(result),
json.encode([
YomiExample(
example: '',
reading: '',
meaning: '''hour, o'clock, (specified) time, when ..., during ...'''
),
YomiExample(
example: '時価',
reading: 'ジカ',
meaning: 'current value, price, market value'
),
YomiExample(
example: '零時',
reading: 'レイジ',
meaning: '''twelve o'clock, midnight, noon'''
),
YomiExample(
example: '平時',
reading: 'ヘイジ',
meaning: 'peacetime, time of peace, ordinary times, normal times'
),
])
);
});
test('getKunyomiExamples', () {
final result = getKunyomiExamples(kanjiPage);
expect(
json.encode(result),
json.encode([
YomiExample(
example: '',
reading: 'とき',
meaning: 'time, hour, moment, occasion, case, chance, opportunity, season, the times, the age, the day, tense'
),
YomiExample(
example: '時折',
reading: 'ときおり',
meaning: 'sometimes'
),
YomiExample(
example: '切り替え時',
reading: 'きりかえとき',
meaning: 'time to switch over, response time'
),
YomiExample(
example: '逢魔が時',
reading: 'おうまがとき',
meaning: '''twilight, time for disasters (similar to 'the witching hour' but not midnight)'''
),
])
);
});
test('getRadical', () {
final result = getRadical(kanjiPage);
expect(
json.encode(result),
json.encode(Radical(
symbol: '',
meaning: 'sun, day'
))
);
});
test('getParts', () {
final result = getParts(kanjiPage);
expect(result, ['', '', '']);
});
test('getSvgUri', () {
final result = getSvgUri(kanjiPage);
expect(result, 'https://d1w6u4xc3l95km.cloudfront.net/kanji-2015-03/06642.svg');
});
test('getGifUri', () {
final result = getGifUri(kanjiPage);
expect(result, 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/3c.gif');
});
test('getNewspaperFrequencyRank', () {
final result = getNewspaperFrequencyRank(kanjiPage);
expect(result, 16);
});
test('parseKanjiPageData', () {
final result = parseKanjiPageData(kanjiPage, '');
final expectedResult = KanjiResult();
expectedResult.query = '';
expectedResult.found = true;
expectedResult.taughtIn = 'grade 2';
expectedResult.jlptLevel = 'N5';
expectedResult.newspaperFrequencyRank = 16;
expectedResult.strokeCount = 10;
expectedResult.meaning = 'time, hour';
expectedResult.kunyomi = ['とき', '-どき'];
expectedResult.onyomi = [''];
expectedResult.onyomiExamples =
[
YomiExample(
example: '',
reading: '',
meaning: '''hour, o'clock, (specified) time, when ..., during ...'''
),
YomiExample(
example: '時価',
reading: 'ジカ',
meaning: 'current value, price, market value'
),
YomiExample(
example: '零時',
reading: 'レイジ',
meaning: '''twelve o'clock, midnight, noon'''
),
YomiExample(
example: '平時',
reading: 'ヘイジ',
meaning: 'peacetime, time of peace, ordinary times, normal times'
),
];
expectedResult.kunyomiExamples =
[
YomiExample(
example: '',
reading: 'とき',
meaning: 'time, hour, moment, occasion, case, chance, opportunity, season, the times, the age, the day, tense'
),
YomiExample(
example: '時折',
reading: 'ときおり',
meaning: 'sometimes'
),
YomiExample(
example: '切り替え時',
reading: 'きりかえとき',
meaning: 'time to switch over, response time'
),
YomiExample(
example: '逢魔が時',
reading: 'おうまがとき',
meaning: '''twilight, time for disasters (similar to 'the witching hour' but not midnight)'''
),
];
expectedResult.radical =
Radical(
symbol: '',
meaning: 'sun, day'
);
expectedResult.parts = ['', '', ''];
expectedResult.strokeOrderDiagramUri = 'https://classic.jisho.org/static/images/stroke_diagrams/26178_frames.png';
expectedResult.strokeOrderSvgUri = 'https://d1w6u4xc3l95km.cloudfront.net/kanji-2015-03/06642.svg';
expectedResult.strokeOrderGifUri = 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/6642.gif';
expectedResult.uri = 'https://jisho.org/search/%E6%99%82%23kanji';
expect(
json.encode(result),
json.encode(expectedResult)
);
});
/* KANJI SEARCH FUNCTION TESTS END */
}

View File

@ -3,7 +3,6 @@ import 'package:path/path.dart' as path;
import 'dart:convert'; import 'dart:convert';
import 'package:unofficial_jisho_api/unofficial_jisho_api.dart'; import 'package:unofficial_jisho_api/unofficial_jisho_api.dart';
import 'local_function_test_cases.dart' show test_local_functions;
import 'package:test/test.dart'; import 'package:test/test.dart';
final jisho = JishoApi(); final jisho = JishoApi();
@ -26,11 +25,7 @@ void runTestCases(List<String> testCaseFiles, Function apiFunction) async {
} }
void main() async { void main() async {
await test_local_functions();
await runTestCases(getFilePaths('kanji_test_cases'), jisho.searchForKanji); await runTestCases(getFilePaths('kanji_test_cases'), jisho.searchForKanji);
await runTestCases(getFilePaths('example_test_cases'), jisho.searchForExamples); await runTestCases(getFilePaths('example_test_cases'), jisho.searchForExamples);
await runTestCases(getFilePaths('phrase_scrape_test_cases'), jisho.scrapeForPhrase); await runTestCases(getFilePaths('phrase_scrape_test_cases'), jisho.scrapeForPhrase);
} }