This commit is contained in:
Oystein Kristoffer Tveit 2020-06-26 18:14:00 +02:00
parent e119a81960
commit 93f1a49c89
8 changed files with 262 additions and 301 deletions

View File

@ -1,8 +1,9 @@
/// This library provides search functions for searching/scraping Jisho.org. /// This library provides search functions for searching/scraping Jisho.org.
/// ///
/// It provides a built-in http client and performs async requests to the server /// It provides a built-in http client and performs async requests to the server
/// for different types of requests. /// for different types of requests.
library unofficial_jisho_api; library unofficial_jisho_api;
import 'dart:convert'; import 'dart:convert';
import 'package:http/http.dart' as http; import 'package:http/http.dart' as http;
@ -13,28 +14,34 @@ import './src/phraseScrape.dart';
import './src/phraseSearch.dart'; import './src/phraseSearch.dart';
/// Query the official Jisho API for a word or phrase /// Query the official Jisho API for a word or phrase
/// ///
/// See https://jisho.org/forum/54fefc1f6e73340b1f160000-is-there-any-kind-of-search-api /// See https://jisho.org/forum/54fefc1f6e73340b1f160000-is-there-any-kind-of-search-api
/// for discussion about the official API. /// for discussion about the official API.
Future<JishoAPIResult> searchForPhrase(String phrase) async { Future<JishoAPIResult> searchForPhrase(String phrase) async {
final uri = uriForPhraseSearch(phrase); final uri = uriForPhraseSearch(phrase);
return await http.get(uri).then((response) => JishoAPIResult.fromJson(jsonDecode(response.body))); return await http
.get(uri)
.then((response) => JishoAPIResult.fromJson(jsonDecode(response.body)));
} }
/// Scrape Jisho.org for information about a kanji character. /// Scrape Jisho.org for information about a kanji character.
Future<KanjiResult> searchForKanji(String kanji) async { Future<KanjiResult> searchForKanji(String kanji) async {
final uri = uriForKanjiSearch(kanji); final uri = uriForKanjiSearch(kanji);
return http.get(uri).then((response) => parseKanjiPageData(response.body, kanji)); return http
.get(uri)
.then((response) => parseKanjiPageData(response.body, kanji));
} }
/// Scrape Jisho.org for examples. /// Scrape Jisho.org for examples.
Future<ExampleResults> searchForExamples(String phrase) async { Future<ExampleResults> searchForExamples(String phrase) async {
final uri = uriForExampleSearch(phrase); final uri = uriForExampleSearch(phrase);
return http.get(uri).then((response) => parseExamplePageData(response.body, phrase)); return http
.get(uri)
.then((response) => parseExamplePageData(response.body, phrase));
} }
/// Scrape the word page for a word/phrase. /// Scrape the word page for a word/phrase.
/// ///
/// This allows you to get some information that isn't provided by the official API, such as /// This allows you to get some information that isn't provided by the official API, such as
/// part-of-speech and JLPT level. However, the official API should be preferred /// part-of-speech and JLPT level. However, the official API should be preferred
/// if it has the information you need. This function scrapes https://jisho.org/word/XXX. /// if it has the information you need. This function scrapes https://jisho.org/word/XXX.
@ -50,4 +57,4 @@ Future<PhrasePageScrapeResult> scrapeForPhrase(String phrase) async {
); );
} }
return parsePhrasePageData(response.body, phrase); return parsePhrasePageData(response.body, phrase);
} }

View File

@ -1,11 +1,12 @@
/// This library provides parsing functions for html pages from Jisho.org, /// This library provides parsing functions for html pages from Jisho.org,
/// and the associated URI-producing functions. /// and the associated URI-producing functions.
/// ///
/// This might be useful for using a CORS proxy, or if you have your own system/library /// This might be useful for using a CORS proxy, or if you have your own system/library
/// for providing HTML. /// for providing HTML.
library unofficial_jisho_parser; library unofficial_jisho_parser;
export './src/exampleSearch.dart' show uriForExampleSearch, parseExamplePageData; export './src/exampleSearch.dart'
show uriForExampleSearch, parseExamplePageData;
export './src/kanjiSearch.dart' show uriForKanjiSearch, parseKanjiPageData; export './src/kanjiSearch.dart' show uriForKanjiSearch, parseKanjiPageData;
export './src/phraseScrape.dart' show uriForPhraseScrape, parsePhrasePageData; export './src/phraseScrape.dart' show uriForPhraseScrape, parsePhrasePageData;
export './src/phraseSearch.dart'; export './src/phraseSearch.dart';

View File

@ -1,3 +1,4 @@
const String JISHO_API = 'https://jisho.org/api/v1/search/words'; const String JISHO_API = 'https://jisho.org/api/v1/search/words';
const String SCRAPE_BASE_URI = 'https://jisho.org/search/'; const String SCRAPE_BASE_URI = 'https://jisho.org/search/';
const String STROKE_ORDER_DIAGRAM_BASE_URI = 'https://classic.jisho.org/static/images/stroke_diagrams/'; const String STROKE_ORDER_DIAGRAM_BASE_URI =
'https://classic.jisho.org/static/images/stroke_diagrams/';

View File

@ -19,21 +19,22 @@ List<Element> _getChildrenAndSymbols(Element ul) {
List<Element> result = []; List<Element> result = [];
for (var element in ulChildren) { for (var element in ulChildren) {
if (element.text != ulText.substring(offsetPointer, offsetPointer + element.text.length)){ if (element.text !=
ulText.substring(offsetPointer, offsetPointer + element.text.length)) {
var symbols = ''; var symbols = '';
while (element.text.substring(0,1) != ulCharArray[offsetPointer]) { while (element.text.substring(0, 1) != ulCharArray[offsetPointer]) {
symbols += ulCharArray[offsetPointer]; symbols += ulCharArray[offsetPointer];
offsetPointer++; offsetPointer++;
} }
final symbolElement = Element.html('<span>$symbols</span>'); final symbolElement = Element.html('<span>$symbols</span>');
result.add(symbolElement); result.add(symbolElement);
} }
offsetPointer += element.text.length; offsetPointer += element.text.length;
result.add(element); result.add(element);
} }
if (offsetPointer + 1 != ulText.length){ if (offsetPointer + 1 != ulText.length) {
final symbols = ulText.substring(offsetPointer, ulText.length-1); final symbols = ulText.substring(offsetPointer, ulText.length - 1);
final symbolElement = Element.html('<span>$symbols</span>'); final symbolElement = Element.html('<span>$symbols</span>');
result.add(symbolElement); result.add(symbolElement);
} }
return result; return result;
@ -122,4 +123,4 @@ ExampleResults parseExamplePageData(String pageHtml, String phrase) {
uri: uriForExampleSearch(phrase), uri: uriForExampleSearch(phrase),
phrase: phrase, phrase: phrase,
); );
} }

View File

@ -9,7 +9,7 @@ const _onyomiLocatorSymbol = 'On';
const _kunyomiLocatorSymbol = 'Kun'; const _kunyomiLocatorSymbol = 'Kun';
String _removeNewlines(String str) { String _removeNewlines(String str) {
return str.replaceAll(RegExp(r'(?:\r|\n)') , '').trim(); return str.replaceAll(RegExp(r'(?:\r|\n)'), '').trim();
} }
/// Provides the URI for a kanji search /// Provides the URI for a kanji search
@ -22,7 +22,8 @@ String _getUriForStrokeOrderDiagram(String kanji) {
} }
bool _containsKanjiGlyph(String pageHtml, String kanji) { bool _containsKanjiGlyph(String pageHtml, String kanji) {
final kanjiGlyphToken = '<h1 class="character" data-area-name="print" lang="ja">$kanji</h1>'; final kanjiGlyphToken =
'<h1 class="character" data-area-name="print" lang="ja">$kanji</h1>';
return pageHtml.contains(kanjiGlyphToken); return pageHtml.contains(kanjiGlyphToken);
} }
@ -31,15 +32,20 @@ String _getStringBetweenIndicies(String data, int startIndex, int endIndex) {
return _removeNewlines(result).trim(); return _removeNewlines(result).trim();
} }
String _getStringBetweenStrings(String data, String startString, String endString) { String _getStringBetweenStrings(
final regex = RegExp('${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}', dotAll: true); String data, String startString, String endString) {
final regex = RegExp(
'${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}',
dotAll: true);
final match = regex.allMatches(data).toList(); final match = regex.allMatches(data).toList();
return match.isNotEmpty ? match[0].group(1).toString() : null; return match.isNotEmpty ? match[0].group(1).toString() : null;
} }
int _getIntBetweenStrings(String pageHtml, String startString, String endString) { int _getIntBetweenStrings(
final stringBetweenStrings = _getStringBetweenStrings(pageHtml, startString, endString); String pageHtml, String startString, String endString) {
final stringBetweenStrings =
_getStringBetweenStrings(pageHtml, startString, endString);
return int.parse(stringBetweenStrings); return int.parse(stringBetweenStrings);
} }
@ -59,7 +65,8 @@ List<String> _parseAnchorsToArray(String str) {
} }
List<String> _getYomi(String pageHtml, String yomiLocatorSymbol) { List<String> _getYomi(String pageHtml, String yomiLocatorSymbol) {
final yomiSection = _getStringBetweenStrings(pageHtml, '<dt>$yomiLocatorSymbol:</dt>', '</dl>'); final yomiSection = _getStringBetweenStrings(
pageHtml, '<dt>$yomiLocatorSymbol:</dt>', '</dl>');
return _parseAnchorsToArray(yomiSection ?? ''); return _parseAnchorsToArray(yomiSection ?? '');
} }
@ -73,13 +80,15 @@ List<String> _getOnyomi(String pageHtml) {
List<YomiExample> _getYomiExamples(String pageHtml, String yomiLocatorSymbol) { List<YomiExample> _getYomiExamples(String pageHtml, String yomiLocatorSymbol) {
final locatorString = '<h2>$yomiLocatorSymbol reading compounds</h2>'; final locatorString = '<h2>$yomiLocatorSymbol reading compounds</h2>';
final exampleSection = _getStringBetweenStrings(pageHtml, locatorString, '</ul>'); final exampleSection =
if (exampleSection==null) { _getStringBetweenStrings(pageHtml, locatorString, '</ul>');
if (exampleSection == null) {
return null; return null;
} }
final regex = RegExp(r'<li>(.*?)<\/li>', dotAll: true); final regex = RegExp(r'<li>(.*?)<\/li>', dotAll: true);
final regexResults = _getAllGlobalGroupMatches(exampleSection, regex).map((s) => s.trim()); final regexResults =
_getAllGlobalGroupMatches(exampleSection, regex).map((s) => s.trim());
final examples = regexResults.map((regexResult) { final examples = regexResults.map((regexResult) {
final examplesLines = regexResult.split('\n').map((s) => s.trim()).toList(); final examplesLines = regexResult.split('\n').map((s) => s.trim()).toList();
@ -111,17 +120,20 @@ Radical _getRadical(String pageHtml) {
radicalMeaningEndString, radicalMeaningEndString,
).trim(); ).trim();
if (radicalMeaning!=null) { if (radicalMeaning != null) {
final radicalMeaningStartIndex = pageHtml.indexOf(radicalMeaningStartString); final radicalMeaningStartIndex =
pageHtml.indexOf(radicalMeaningStartString);
final radicalMeaningEndIndex = pageHtml.indexOf( final radicalMeaningEndIndex = pageHtml.indexOf(
radicalMeaningEndString, radicalMeaningEndString,
radicalMeaningStartIndex, radicalMeaningStartIndex,
); );
final radicalSymbolStartIndex = radicalMeaningEndIndex + radicalMeaningEndString.length; final radicalSymbolStartIndex =
radicalMeaningEndIndex + radicalMeaningEndString.length;
const radicalSymbolEndString = '</span>'; const radicalSymbolEndString = '</span>';
final radicalSymbolEndIndex = pageHtml.indexOf(radicalSymbolEndString, radicalSymbolStartIndex); final radicalSymbolEndIndex =
pageHtml.indexOf(radicalSymbolEndString, radicalSymbolStartIndex);
final radicalSymbolsString = _getStringBetweenIndicies( final radicalSymbolsString = _getStringBetweenIndicies(
pageHtml, pageHtml,
@ -131,23 +143,19 @@ Radical _getRadical(String pageHtml) {
if (radicalSymbolsString.length > 1) { if (radicalSymbolsString.length > 1) {
final radicalForms = radicalSymbolsString final radicalForms = radicalSymbolsString
.substring(1) .substring(1)
.replaceAll('(', '') .replaceAll('(', '')
.replaceAll(')', '') .replaceAll(')', '')
.trim() .trim()
.split(', '); .split(', ');
return Radical( return Radical(
symbol: radicalSymbolsString[0], symbol: radicalSymbolsString[0],
forms: radicalForms ?? [], forms: radicalForms ?? [],
meaning: radicalMeaning meaning: radicalMeaning);
);
} }
return Radical ( return Radical(symbol: radicalSymbolsString, meaning: radicalMeaning);
symbol: radicalSymbolsString,
meaning: radicalMeaning
);
} }
return null; return null;
@ -178,14 +186,19 @@ String _getSvgUri(String pageHtml) {
String _getGifUri(String kanji) { String _getGifUri(String kanji) {
final unicodeString = kanji.codeUnitAt(0).toRadixString(16); final unicodeString = kanji.codeUnitAt(0).toRadixString(16);
final fileName = '$unicodeString.gif'; final fileName = '$unicodeString.gif';
final animationUri = 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/$fileName'; final animationUri =
'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/$fileName';
return animationUri; return animationUri;
} }
int _getNewspaperFrequencyRank(String pageHtml) { int _getNewspaperFrequencyRank(String pageHtml) {
final frequencySection = _getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>'); final frequencySection =
return (frequencySection != null) ? int.parse(_getStringBetweenStrings(frequencySection, '<strong>', '</strong>')) : null; _getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>');
return (frequencySection != null)
? int.parse(
_getStringBetweenStrings(frequencySection, '<strong>', '</strong>'))
: null;
} }
/// Parses a jisho kanji search page to an object /// Parses a jisho kanji search page to an object
@ -193,15 +206,21 @@ KanjiResult parseKanjiPageData(String pageHtml, String kanji) {
final result = KanjiResult(); final result = KanjiResult();
result.query = kanji; result.query = kanji;
result.found = _containsKanjiGlyph(pageHtml, kanji); result.found = _containsKanjiGlyph(pageHtml, kanji);
if (result.found==false) { if (result.found == false) {
return result; return result;
} }
result.taughtIn = _getStringBetweenStrings(pageHtml, 'taught in <strong>', '</strong>'); result.taughtIn =
result.jlptLevel = _getStringBetweenStrings(pageHtml, 'JLPT level <strong>', '</strong>'); _getStringBetweenStrings(pageHtml, 'taught in <strong>', '</strong>');
result.jlptLevel =
_getStringBetweenStrings(pageHtml, 'JLPT level <strong>', '</strong>');
result.newspaperFrequencyRank = _getNewspaperFrequencyRank(pageHtml); result.newspaperFrequencyRank = _getNewspaperFrequencyRank(pageHtml);
result.strokeCount = _getIntBetweenStrings(pageHtml, '<strong>', '</strong> strokes'); result.strokeCount =
result.meaning = _htmlUnescape.convert(_removeNewlines(_getStringBetweenStrings(pageHtml, '<div class="kanji-details__main-meanings">', '</div>')).trim()); _getIntBetweenStrings(pageHtml, '<strong>', '</strong> strokes');
result.meaning = _htmlUnescape.convert(_removeNewlines(
_getStringBetweenStrings(
pageHtml, '<div class="kanji-details__main-meanings">', '</div>'))
.trim());
result.kunyomi = _getKunyomi(pageHtml) ?? []; result.kunyomi = _getKunyomi(pageHtml) ?? [];
result.onyomi = _getOnyomi(pageHtml) ?? []; result.onyomi = _getOnyomi(pageHtml) ?? [];
result.onyomiExamples = _getOnyomiExamples(pageHtml) ?? []; result.onyomiExamples = _getOnyomiExamples(pageHtml) ?? [];
@ -213,4 +232,4 @@ KanjiResult parseKanjiPageData(String pageHtml, String kanji) {
result.strokeOrderGifUri = _getGifUri(kanji); result.strokeOrderGifUri = _getGifUri(kanji);
result.uri = uriForKanjiSearch(kanji); result.uri = uriForKanjiSearch(kanji);
return result; return result;
} }

View File

@ -7,19 +7,10 @@ class YomiExample {
String reading; String reading;
String meaning; String meaning;
YomiExample({ YomiExample({this.example, this.reading, this.meaning});
this.example,
this.reading,
this.meaning
});
Map<String, String> toJson() => Map<String, String> toJson() =>
{ {'example': example, 'reading': reading, 'meaning': meaning};
'example': example,
'reading': reading,
'meaning': meaning
};
} }
class Radical { class Radical {
@ -27,19 +18,10 @@ class Radical {
List<String> forms; List<String> forms;
String meaning; String meaning;
Radical({ Radical({this.symbol, this.forms, this.meaning});
this.symbol,
this.forms,
this.meaning
});
Map<String, dynamic> toJson() => Map<String, dynamic> toJson() =>
{ {'symbol': symbol, 'forms': forms, 'meaning': meaning};
'symbol': symbol,
'forms': forms,
'meaning': meaning
};
} }
class KanjiResult { class KanjiResult {
@ -62,25 +44,24 @@ class KanjiResult {
String strokeOrderGifUri; String strokeOrderGifUri;
String uri; String uri;
KanjiResult({ KanjiResult(
this.query, {this.query,
this.found, this.found,
this.taughtIn, this.taughtIn,
this.jlptLevel, this.jlptLevel,
this.newspaperFrequencyRank, this.newspaperFrequencyRank,
this.strokeCount, this.strokeCount,
this.meaning, this.meaning,
this.kunyomi, this.kunyomi,
this.onyomi, this.onyomi,
this.kunyomiExamples, this.kunyomiExamples,
this.onyomiExamples, this.onyomiExamples,
this.radical, this.radical,
this.parts, this.parts,
this.strokeOrderDiagramUri, this.strokeOrderDiagramUri,
this.strokeOrderSvgUri, this.strokeOrderSvgUri,
this.strokeOrderGifUri, this.strokeOrderGifUri,
this.uri this.uri});
});
Map<String, dynamic> toJson() { Map<String, dynamic> toJson() {
return { return {
@ -113,16 +94,10 @@ class ExampleSentencePiece {
String lifted; String lifted;
String unlifted; String unlifted;
ExampleSentencePiece({ ExampleSentencePiece({this.lifted, this.unlifted});
this.lifted,
this.unlifted
});
Map<String, dynamic> toJson() { Map<String, dynamic> toJson() {
return { return {'lifted': lifted, 'unlifted': unlifted};
'lifted': lifted,
'unlifted': unlifted
};
} }
} }
@ -132,20 +107,10 @@ class ExampleResultData {
String english; String english;
List<ExampleSentencePiece> pieces; List<ExampleSentencePiece> pieces;
ExampleResultData({ ExampleResultData({this.english, this.kanji, this.kana, this.pieces});
this.english,
this.kanji,
this.kana,
this.pieces
});
Map<String, dynamic> toJson() { Map<String, dynamic> toJson() {
return { return {'english': english, 'kanji': kanji, 'kana': kana, 'pieces': pieces};
'english': english,
'kanji': kanji,
'kana': kana,
'pieces': pieces
};
} }
} }
@ -156,13 +121,7 @@ class ExampleResults {
List<ExampleResultData> results; List<ExampleResultData> results;
String phrase; String phrase;
ExampleResults({ ExampleResults({this.query, this.found, this.results, this.uri, this.phrase});
this.query,
this.found,
this.results,
this.uri,
this.phrase
});
Map<String, dynamic> toJson() { Map<String, dynamic> toJson() {
return { return {
@ -184,17 +143,10 @@ class PhraseScrapeSentence {
String japanese; String japanese;
List<ExampleSentencePiece> pieces; List<ExampleSentencePiece> pieces;
PhraseScrapeSentence ({ PhraseScrapeSentence({this.english, this.japanese, this.pieces});
this.english,
this.japanese,
this.pieces
});
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() =>
'english': english, {'english': english, 'japanese': japanese, 'pieces': pieces};
'japanese': japanese,
'pieces': pieces
};
} }
class PhraseScrapeMeaning { class PhraseScrapeMeaning {
@ -205,38 +157,31 @@ class PhraseScrapeMeaning {
String definitionAbstract; String definitionAbstract;
List<String> tags; List<String> tags;
PhraseScrapeMeaning({ PhraseScrapeMeaning(
this.seeAlsoTerms, {this.seeAlsoTerms,
this.sentences, this.sentences,
this.definition, this.definition,
this.supplemental, this.supplemental,
this.definitionAbstract, this.definitionAbstract,
this.tags this.tags});
});
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {
'seeAlsoTerms': seeAlsoTerms, 'seeAlsoTerms': seeAlsoTerms,
'sentences': sentences, 'sentences': sentences,
'definition': definition, 'definition': definition,
'supplemental': supplemental, 'supplemental': supplemental,
'definitionAbstract': definitionAbstract, 'definitionAbstract': definitionAbstract,
'tags': tags 'tags': tags
}; };
} }
class KanjiKanaPair { class KanjiKanaPair {
String kanji; String kanji;
String kana; String kana;
KanjiKanaPair({ KanjiKanaPair({this.kanji, this.kana});
this.kanji,
this.kana
});
Map<String, String> toJson() => { Map<String, String> toJson() => {'kanji': kanji, 'kana': kana};
'kanji': kanji,
'kana': kana
};
} }
class PhrasePageScrapeResult { class PhrasePageScrapeResult {
@ -248,25 +193,24 @@ class PhrasePageScrapeResult {
List<KanjiKanaPair> otherForms; List<KanjiKanaPair> otherForms;
List<String> notes; List<String> notes;
PhrasePageScrapeResult({ PhrasePageScrapeResult(
this.found, {this.found,
this.query, this.query,
this.uri, this.uri,
this.tags, this.tags,
this.meanings, this.meanings,
this.otherForms, this.otherForms,
this.notes this.notes});
});
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {
'found': found, 'found': found,
'query': query, 'query': query,
'uri': uri, 'uri': uri,
'tags': tags, 'tags': tags,
'meanings': meanings, 'meanings': meanings,
'otherForms': otherForms, 'otherForms': otherForms,
'notes': notes 'notes': notes
}; };
} }
/* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */
@ -279,17 +223,12 @@ class JishoJapaneseWord {
JishoJapaneseWord({this.word, this.reading}); JishoJapaneseWord({this.word, this.reading});
factory JishoJapaneseWord.fromJson(Map<String, dynamic> json){ factory JishoJapaneseWord.fromJson(Map<String, dynamic> json) {
return JishoJapaneseWord( return JishoJapaneseWord(
word: json['word'] as String, word: json['word'] as String, reading: json['reading'] as String);
reading: json['reading'] as String
);
} }
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {'word': word, 'reading': reading};
'word': word,
'reading': reading
};
} }
class JishoSenseLink { class JishoSenseLink {
@ -298,17 +237,12 @@ class JishoSenseLink {
JishoSenseLink({this.text, this.url}); JishoSenseLink({this.text, this.url});
factory JishoSenseLink.fromJson(Map<String, dynamic> json){ factory JishoSenseLink.fromJson(Map<String, dynamic> json) {
return JishoSenseLink( return JishoSenseLink(
text: json['text'] as String, text: json['text'] as String, url: json['url'] as String);
url: json['url'] as String
);
} }
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {'text': text, 'url': url};
'text': text,
'url': url
};
} }
class JishoWordSense { class JishoWordSense {
@ -322,43 +256,51 @@ class JishoWordSense {
List<String> info; List<String> info;
List<dynamic> restrictions; List<dynamic> restrictions;
JishoWordSense({ JishoWordSense(
this.english_definitions, {this.english_definitions,
this.parts_of_speech, this.parts_of_speech,
this.links, this.links,
this.tags, this.tags,
this.see_also, this.see_also,
this.antonyms, this.antonyms,
this.source, this.source,
this.info, this.info,
this.restrictions this.restrictions});
});
factory JishoWordSense.fromJson(Map<String, dynamic> json){ factory JishoWordSense.fromJson(Map<String, dynamic> json) {
return JishoWordSense( return JishoWordSense(
english_definitions: (json['english_definitions'] as List).map((result) => result as String).toList(), english_definitions: (json['english_definitions'] as List)
parts_of_speech: (json['parts_of_speech'] as List).map((result) => result as String).toList(), .map((result) => result as String)
links: (json['links'] as List).map((result) => JishoSenseLink.fromJson(result)).toList(), .toList(),
tags: (json['tags'] as List).map((result) => result as String).toList(), parts_of_speech: (json['parts_of_speech'] as List)
see_also: (json['see_also'] as List).map((result) => result as String).toList(), .map((result) => result as String)
antonyms: (json['antonyms'] as List).map((result) => result as String).toList(), .toList(),
source: json['source'] as List<dynamic>, links: (json['links'] as List)
info: (json['info'] as List).map((result) => result as String).toList(), .map((result) => JishoSenseLink.fromJson(result))
restrictions: json['restrictions'] as List<dynamic> .toList(),
); tags: (json['tags'] as List).map((result) => result as String).toList(),
see_also: (json['see_also'] as List)
.map((result) => result as String)
.toList(),
antonyms: (json['antonyms'] as List)
.map((result) => result as String)
.toList(),
source: json['source'] as List<dynamic>,
info: (json['info'] as List).map((result) => result as String).toList(),
restrictions: json['restrictions'] as List<dynamic>);
} }
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {
'english_definitions': english_definitions, 'english_definitions': english_definitions,
'parts_of_speech': parts_of_speech, 'parts_of_speech': parts_of_speech,
'links': links, 'links': links,
'tags': tags, 'tags': tags,
'see_also': see_also, 'see_also': see_also,
'antonyms': antonyms, 'antonyms': antonyms,
'source': source, 'source': source,
'info': info, 'info': info,
'restrictions': restrictions 'restrictions': restrictions
}; };
} }
class JishoAttribution { class JishoAttribution {
@ -366,25 +308,19 @@ class JishoAttribution {
bool jmnedict; bool jmnedict;
String dbpedia; String dbpedia;
JishoAttribution({ JishoAttribution({this.jmdict, this.jmnedict, this.dbpedia});
this.jmdict,
this.jmnedict,
this.dbpedia
});
factory JishoAttribution.fromJson(Map<String, dynamic> json){ factory JishoAttribution.fromJson(Map<String, dynamic> json) {
return JishoAttribution( return JishoAttribution(
jmdict: (json['jmdict'].toString() == 'true'), jmdict: (json['jmdict'].toString() == 'true'),
jmnedict: (json['jmnedict'].toString() == 'true'), jmnedict: (json['jmnedict'].toString() == 'true'),
dbpedia: (json['dbpedia'].toString() != 'false') ? json['dbpedia'].toString() : null dbpedia: (json['dbpedia'].toString() != 'false')
); ? json['dbpedia'].toString()
: null);
} }
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() =>
'jmdict': jmdict, {'jmdict': jmdict, 'jmnedict': jmnedict, 'dbpedia': dbpedia};
'jmnedict': jmnedict,
'dbpedia': dbpedia
};
} }
class JishoResult { class JishoResult {
@ -396,37 +332,39 @@ class JishoResult {
List<JishoWordSense> senses; List<JishoWordSense> senses;
JishoAttribution attribution; JishoAttribution attribution;
JishoResult({ JishoResult(
this.slug, {this.slug,
this.is_common, this.is_common,
this.tags, this.tags,
this.jlpt, this.jlpt,
this.japanese, this.japanese,
this.senses, this.senses,
this.attribution this.attribution});
});
factory JishoResult.fromJson(Map<String, dynamic> json){ factory JishoResult.fromJson(Map<String, dynamic> json) {
return JishoResult( return JishoResult(
slug: json['slug'] as String, slug: json['slug'] as String,
is_common: json['is_common'] as bool, is_common: json['is_common'] as bool,
tags: (json['tags'] as List).map((result) => result as String).toList(), tags: (json['tags'] as List).map((result) => result as String).toList(),
jlpt: (json['jlpt'] as List).map((result) => result as String).toList(), jlpt: (json['jlpt'] as List).map((result) => result as String).toList(),
japanese: (json['japanese'] as List).map((result) => JishoJapaneseWord.fromJson(result)).toList(), japanese: (json['japanese'] as List)
senses: (json['senses'] as List).map((result) => JishoWordSense.fromJson(result)).toList(), .map((result) => JishoJapaneseWord.fromJson(result))
attribution: JishoAttribution.fromJson(json['attribution']) .toList(),
); senses: (json['senses'] as List)
.map((result) => JishoWordSense.fromJson(result))
.toList(),
attribution: JishoAttribution.fromJson(json['attribution']));
} }
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {
'slug': slug, 'slug': slug,
'is_common': is_common, 'is_common': is_common,
'tags': tags, 'tags': tags,
'jlpt': jlpt, 'jlpt': jlpt,
'japanese': japanese, 'japanese': japanese,
'senses': senses, 'senses': senses,
'attribution': attribution 'attribution': attribution
}; };
} }
class JishoResultMeta { class JishoResultMeta {
@ -434,15 +372,11 @@ class JishoResultMeta {
JishoResultMeta({this.status}); JishoResultMeta({this.status});
factory JishoResultMeta.fromJson(Map<String, dynamic> json){ factory JishoResultMeta.fromJson(Map<String, dynamic> json) {
return JishoResultMeta( return JishoResultMeta(status: json['status'] as int);
status: json['status'] as int
);
} }
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {'status': status};
'status': status
};
} }
class JishoAPIResult { class JishoAPIResult {
@ -451,15 +385,13 @@ class JishoAPIResult {
JishoAPIResult({this.meta, this.data}); JishoAPIResult({this.meta, this.data});
factory JishoAPIResult.fromJson(Map<String, dynamic> json){ factory JishoAPIResult.fromJson(Map<String, dynamic> json) {
return JishoAPIResult( return JishoAPIResult(
meta: JishoResultMeta.fromJson(json['meta']), meta: JishoResultMeta.fromJson(json['meta']),
data: (json['data'] as List).map((result) => JishoResult.fromJson(result)).toList() data: (json['data'] as List)
); .map((result) => JishoResult.fromJson(result))
.toList());
} }
Map<String, dynamic> toJson() => { Map<String, dynamic> toJson() => {'meta': meta.toJson(), 'data': data};
'meta': meta.toJson(), }
'data': data
};
}

View File

@ -21,19 +21,23 @@ List<String> _getMostRecentWordTypes(Element child) {
} }
List<KanjiKanaPair> _getOtherForms(Element child) { List<KanjiKanaPair> _getOtherForms(Element child) {
return child.text.split('') return child.text
.map((s) => s.replaceAll('', '').replaceAll('', '').split(' ')) .split('')
.map((a) => (KanjiKanaPair( kanji: a[0], kana: (a.length == 2) ? a[1] : null ))).toList(); .map((s) => s.replaceAll('', '').replaceAll('', '').split(' '))
.map((a) =>
(KanjiKanaPair(kanji: a[0], kana: (a.length == 2) ? a[1] : null)))
.toList();
} }
List<String> _getNotes(Element child) => child.text.split('\n'); List<String> _getNotes(Element child) => child.text.split('\n');
String _getMeaning(Element child) => child.querySelector('.meaning-meaning').text; String _getMeaning(Element child) =>
child.querySelector('.meaning-meaning').text;
String _getMeaningAbstract(Element child) { String _getMeaningAbstract(Element child) {
final meaningAbstract = child.querySelector('.meaning-abstract'); final meaningAbstract = child.querySelector('.meaning-abstract');
if (meaningAbstract == null) return null; if (meaningAbstract == null) return null;
for (var element in meaningAbstract.querySelectorAll('a')) { for (var element in meaningAbstract.querySelectorAll('a')) {
element.remove(); element.remove();
} }
@ -62,11 +66,14 @@ List<String> _getSeeAlsoTerms(List<String> supplemental) {
} }
List<PhraseScrapeSentence> _getSentences(Element child) { List<PhraseScrapeSentence> _getSentences(Element child) {
final sentenceElements = child.querySelector('.sentences')?.querySelectorAll('.sentence'); final sentenceElements =
child.querySelector('.sentences')?.querySelectorAll('.sentence');
if (sentenceElements == null) return []; if (sentenceElements == null) return [];
final List<PhraseScrapeSentence> sentences = []; final List<PhraseScrapeSentence> sentences = [];
for (var sentenceIndex = 0; sentenceIndex < (sentenceElements?.length ?? 0); sentenceIndex += 1) { for (var sentenceIndex = 0;
sentenceIndex < (sentenceElements?.length ?? 0);
sentenceIndex += 1) {
final sentenceElement = sentenceElements[sentenceIndex]; final sentenceElement = sentenceElements[sentenceIndex];
final english = sentenceElement.querySelector('.english').text; final english = sentenceElement.querySelector('.english').text;
@ -79,20 +86,15 @@ List<PhraseScrapeSentence> _getSentences(Element child) {
final japanese = sentenceElement.text; final japanese = sentenceElement.text;
sentences.add( sentences.add(PhraseScrapeSentence(
PhraseScrapeSentence( english: english, japanese: japanese, pieces: pieces ?? []));
english: english,
japanese: japanese,
pieces: pieces ?? []
)
);
} }
return sentences; return sentences;
} }
PhrasePageScrapeResult _getMeaningsOtherFormsAndNotes(Document document) { PhrasePageScrapeResult _getMeaningsOtherFormsAndNotes(Document document) {
final returnValues = PhrasePageScrapeResult( otherForms: [], notes: [] ); final returnValues = PhrasePageScrapeResult(otherForms: [], notes: []);
final meaningsWrapper = document.querySelector('.meanings-wrapper'); final meaningsWrapper = document.querySelector('.meanings-wrapper');
if (meaningsWrapper == null) return PhrasePageScrapeResult(found: false); if (meaningsWrapper == null) return PhrasePageScrapeResult(found: false);
@ -102,18 +104,17 @@ PhrasePageScrapeResult _getMeaningsOtherFormsAndNotes(Document document) {
final List<PhraseScrapeMeaning> meanings = []; final List<PhraseScrapeMeaning> meanings = [];
var mostRecentWordTypes = []; var mostRecentWordTypes = [];
for (var meaningIndex = 0; meaningIndex < meaningsChildren.length; meaningIndex += 1) { for (var meaningIndex = 0;
meaningIndex < meaningsChildren.length;
meaningIndex += 1) {
final child = meaningsChildren[meaningIndex]; final child = meaningsChildren[meaningIndex];
if (child.className.contains('meaning-tags')) { if (child.className.contains('meaning-tags')) {
mostRecentWordTypes = _getMostRecentWordTypes(child); mostRecentWordTypes = _getMostRecentWordTypes(child);
} else if (mostRecentWordTypes[0] == 'other forms') { } else if (mostRecentWordTypes[0] == 'other forms') {
returnValues.otherForms = _getOtherForms(child); returnValues.otherForms = _getOtherForms(child);
} else if (mostRecentWordTypes[0] == 'notes') { } else if (mostRecentWordTypes[0] == 'notes') {
returnValues.notes = _getNotes(child); returnValues.notes = _getNotes(child);
} else { } else {
final meaning = _getMeaning(child); final meaning = _getMeaning(child);
final meaningAbstract = _getMeaningAbstract(child); final meaningAbstract = _getMeaningAbstract(child);
@ -153,4 +154,4 @@ PhrasePageScrapeResult parsePhrasePageData(String pageHtml, String query) {
result.tags = _getTags(document); result.tags = _getTags(document);
return result; return result;
} }

View File

@ -1,7 +1,6 @@
import './base_uri.dart'; import './base_uri.dart';
/// Provides the URI for a phrase search /// Provides the URI for a phrase search
String uriForPhraseSearch(String phrase) { String uriForPhraseSearch(String phrase) {
return '$JISHO_API?keyword=${Uri.encodeComponent(phrase)}'; return '$JISHO_API?keyword=${Uri.encodeComponent(phrase)}';
} }