oysteikt/unofficial_jisho_api_dart
oysteikt
/
unofficial_jisho_api_dart
mirror of https://github.com/h7x4/unofficial_jisho_api_dart.git
1
0
Fork 0
Code Issues Releases Activity

Rewrite some code by effective dart

This commit is contained in:
Oystein Kristoffer Tveit 2020-06-26 01:59:23 +02:00
parent d6509e6392
commit 093ae02c56
16 changed files with 187 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
README.md
View File

@ -31,22 +31,22 @@ import 'package:unofficial_jisho_api/api.dart' as jisho;
void main() async { void main() async {
await jisho.searchForKanji('語').then((result) { await jisho.searchForKanji('語').then((result) {
print('Found: ' + result.found.toString()); print('Found: ${result.found}');
print('Taught in: ' + result.taughtIn); print('Taught in: ${result.taughtIn}');
print('JLPT level: ' + result.jlptLevel); print('JLPT level: ${result.jlptLevel}');
print('Newspaper frequency rank: ' + result.newspaperFrequencyRank.toString()); print('Newspaper frequency rank: ${result.newspaperFrequencyRank}');
print('Stroke count: ' + result.strokeCount.toString()); print('Stroke count: ${result.strokeCount}');
print('Meaning: ' + result.meaning); print('Meaning: ${result.meaning}');
print('Kunyomi: ' + jsonEncode(result.kunyomi)); print('Kunyomi: ${jsonEncode(result.kunyomi)}');
print('Kunyomi example: ' + jsonEncode(result.kunyomiExamples[0])); print('Kunyomi example: ${jsonEncode(result.kunyomiExamples[0])}');
print('Onyomi: ' + jsonEncode(result.onyomi)); print('Onyomi: ${jsonEncode(result.onyomi)}');
print('Onyomi example: ' + jsonEncode(result.onyomiExamples[0])); print('Onyomi example: ${jsonEncode(result.onyomiExamples[0])}');
print('Radical: ' + jsonEncode(result.radical)); print('Radical: ${jsonEncode(result.radical)}');
print('Parts: ' + jsonEncode(result.parts)); print('Parts: ${jsonEncode(result.parts)}');
print('Stroke order diagram: ' + result.strokeOrderDiagramUri); print('Stroke order diagram: ${result.strokeOrderDiagramUri}');
print('Stroke order SVG: ' + result.strokeOrderSvgUri); print('Stroke order SVG: ${result.strokeOrderSvgUri}');
print('Stroke order GIF: ' + result.strokeOrderGifUri); print('Stroke order GIF: ${result.strokeOrderGifUri}');
print('Jisho Uri: ' + result.uri); print('Jisho Uri: ${result.uri}');
}); });
} }
``` ```
@ -80,10 +80,10 @@ import 'package:unofficial_jisho_api/api.dart' as jisho;
void main() async { void main() async {
await jisho.searchForExamples('日').then((result) { await jisho.searchForExamples('日').then((result) {
print('Jisho Uri: ' + result.uri); print('Jisho Uri: ${result.uri}');
print(''); print('');
for (int i = 0; i < 3; i++) { for (var i = 0; i < 3; i++) {
var example = result.results[i]; var example = result.results[i];
print(example.kanji); print(example.kanji);
print(example.kana); print(example.kana);
@ -124,7 +124,7 @@ This scrapes the word/phrase page on Jisho.org. This can get you some data that
```dart ```dart
import 'dart:convert'; import 'dart:convert';
import 'package:unofficial_jisho_api/api.dart' as jisho; import 'package:unofficial_jisho_api/api.dart' as jisho;
final encoder = JsonEncoder.withIndent(' '); final JsonEncoder encoder = JsonEncoder.withIndent(' ');
void main() async { void main() async {
await jisho.scrapeForPhrase('谷').then((data) { await jisho.scrapeForPhrase('谷').then((data) {
@ -188,18 +188,18 @@ You can provide the HTML responses from Jisho yourself. This can be useful if yo
### Parse kanji page HTML ### Parse kanji page HTML
```dart ```dart
import 'package:http/http.dart' as http;
import 'dart:convert'; import 'dart:convert';
import 'package:http/http.dart' as http;
import 'package:unofficial_jisho_api/parser.dart' as jisho_parser; import 'package:unofficial_jisho_api/parser.dart' as jisho_parser;
final encoder = JsonEncoder.withIndent(' '); final JsonEncoder encoder = JsonEncoder.withIndent(' ');
const SEARCH_KANJI = '車'; const String searchKanji = '車';
final SEARCH_URI = jisho_parser.uriForKanjiSearch(SEARCH_KANJI); final String searchURI = jisho_parser.uriForKanjiSearch(searchKanji);
void main() async { void main() async {
await http.get(SEARCH_URI).then((result) { await http.get(searchURI).then((result) {
final parsedResult = jisho_parser.parseKanjiPageData(result.body, SEARCH_KANJI); final parsedResult = jisho_parser.parseKanjiPageData(result.body, searchKanji);
print('JLPT level: ${parsedResult.jlptLevel}'); print('JLPT level: ${parsedResult.jlptLevel}');
print('Stroke count: ${parsedResult.strokeCount}'); print('Stroke count: ${parsedResult.strokeCount}');
print('Meaning: ${parsedResult.meaning}'); print('Meaning: ${parsedResult.meaning}');
@ -210,18 +210,18 @@ void main() async {
### Parse example page HTML ### Parse example page HTML
```dart ```dart
import 'package:http/http.dart' as http;
import 'dart:convert'; import 'dart:convert';
import 'package:http/http.dart' as http;
import 'package:unofficial_jisho_api/parser.dart' as jisho_parser; import 'package:unofficial_jisho_api/parser.dart' as jisho_parser;
final encoder = JsonEncoder.withIndent(' '); final JsonEncoder encoder = JsonEncoder.withIndent(' ');
const SEARCH_EXAMPLE = '保護者'; const String searchExample = '保護者';
final SEARCH_URI = jisho_parser.uriForExampleSearch(SEARCH_EXAMPLE); final String searchURI = jisho_parser.uriForExampleSearch(searchExample);
void main() async { void main() async {
await http.get(SEARCH_URI).then((result) { await http.get(searchURI).then((result) {
final parsedResult = jisho_parser.parseExamplePageData(result.body, SEARCH_EXAMPLE); final parsedResult = jisho_parser.parseExamplePageData(result.body, searchExample);
print('English: ${parsedResult.results[0].english}'); print('English: ${parsedResult.results[0].english}');
print('Kanji ${parsedResult.results[0].kanji}'); print('Kanji ${parsedResult.results[0].kanji}');
print('Kana: ${parsedResult.results[0].kana}'); print('Kana: ${parsedResult.results[0].kana}');
@ -232,19 +232,19 @@ void main() async {
### Parse phrase page HTML ### Parse phrase page HTML
```dart ```dart
import 'package:http/http.dart' as http;
import 'dart:convert'; import 'dart:convert';
import 'package:http/http.dart' as http;
import 'package:unofficial_jisho_api/parser.dart' as jisho_parser; import 'package:unofficial_jisho_api/parser.dart' as jisho_parser;
final encoder = JsonEncoder.withIndent(' '); final JsonEncoder encoder = JsonEncoder.withIndent(' ');
const SEARCH_EXAMPLE = '保護者'; const String searchExample = '保護者';
final SEARCH_URI = jisho_parser.uriForPhraseScrape(SEARCH_EXAMPLE); final String searchURI = jisho_parser.uriForPhraseScrape(searchExample);
void main() async { void main() async {
await http.get(SEARCH_URI).then((result) { await http.get(searchURI).then((result) {
final parsedResult = jisho_parser.parsePhrasePageData(result.body, SEARCH_EXAMPLE); final parsedResult = jisho_parser.parsePhrasePageData(result.body, searchExample);
print(encoder.convert(parsedResult)); print(encoder.convert(parsedResult));
}); });
} }

3
analysis_options.yaml
View File

@ -1,7 +1,8 @@
# Defines a default set of lint rules enforced for # Defines a default set of lint rules enforced for
# projects at Google. For details and rationale, # projects at Google. For details and rationale,
# see https://github.com/dart-lang/pedantic#enabled-lints. # see https://github.com/dart-lang/pedantic#enabled-lints.
include: package:pedantic/analysis_options.yaml # include: package:pedantic/analysis_options.yaml
include: package:effective_dart/analysis_options.yaml
# For lint rules and documentation, see http://dart-lang.github.io/linter/lints. # For lint rules and documentation, see http://dart-lang.github.io/linter/lints.
# Uncomment to specify additional rules. # Uncomment to specify additional rules.

2
example/api/example_search.dart
View File

@ -3,7 +3,7 @@ import 'package:unofficial_jisho_api/api.dart' as jisho;
void main() async { void main() async {
await jisho.searchForExamples('日').then((result) { await jisho.searchForExamples('日').then((result) {
print('Jisho Uri: ' + result.uri); print('Jisho Uri: ${result.uri}');
print(''); print('');
for (var i = 0; i < 3; i++) { for (var i = 0; i < 3; i++) {

32
example/api/kanji_search.dart
View File

@ -3,21 +3,21 @@ import 'package:unofficial_jisho_api/api.dart' as jisho;
void main() async { void main() async {
await jisho.searchForKanji('語').then((result) { await jisho.searchForKanji('語').then((result) {
print('Found: ' + result.found.toString()); print('Found: ${result.found}');
print('Taught in: ' + result.taughtIn); print('Taught in: ${result.taughtIn}');
print('JLPT level: ' + result.jlptLevel); print('JLPT level: ${result.jlptLevel}');
print('Newspaper frequency rank: ' + result.newspaperFrequencyRank.toString()); print('Newspaper frequency rank: ${result.newspaperFrequencyRank}');
print('Stroke count: ' + result.strokeCount.toString()); print('Stroke count: ${result.strokeCount}');
print('Meaning: ' + result.meaning); print('Meaning: ${result.meaning}');
print('Kunyomi: ' + jsonEncode(result.kunyomi)); print('Kunyomi: ${jsonEncode(result.kunyomi)}');
print('Kunyomi example: ' + jsonEncode(result.kunyomiExamples[0])); print('Kunyomi example: ${jsonEncode(result.kunyomiExamples[0])}');
print('Onyomi: ' + jsonEncode(result.onyomi)); print('Onyomi: ${jsonEncode(result.onyomi)}');
print('Onyomi example: ' + jsonEncode(result.onyomiExamples[0])); print('Onyomi example: ${jsonEncode(result.onyomiExamples[0])}');
print('Radical: ' + jsonEncode(result.radical)); print('Radical: ${jsonEncode(result.radical)}');
print('Parts: ' + jsonEncode(result.parts)); print('Parts: ${jsonEncode(result.parts)}');
print('Stroke order diagram: ' + result.strokeOrderDiagramUri); print('Stroke order diagram: ${result.strokeOrderDiagramUri}');
print('Stroke order SVG: ' + result.strokeOrderSvgUri); print('Stroke order SVG: ${result.strokeOrderSvgUri}');
print('Stroke order GIF: ' + result.strokeOrderGifUri); print('Stroke order GIF: ${result.strokeOrderGifUri}');
print('Jisho Uri: ' + result.uri); print('Jisho Uri: ${result.uri}');
}); });
} }

2
example/api/phrase_scrape.dart
View File

@ -1,6 +1,6 @@
import 'dart:convert'; import 'dart:convert';
import 'package:unofficial_jisho_api/api.dart' as jisho; import 'package:unofficial_jisho_api/api.dart' as jisho;
final encoder = JsonEncoder.withIndent(' '); final JsonEncoder encoder = JsonEncoder.withIndent(' ');
void main() async { void main() async {
await jisho.scrapeForPhrase('谷').then((data) { await jisho.scrapeForPhrase('谷').then((data) {

2
example/api/phrase_search.dart
View File

@ -1,6 +1,6 @@
import 'dart:convert'; import 'dart:convert';
import 'package:unofficial_jisho_api/api.dart' as jisho; import 'package:unofficial_jisho_api/api.dart' as jisho;
final encoder = JsonEncoder.withIndent(' '); final JsonEncoder encoder = JsonEncoder.withIndent(' ');
void main() async { void main() async {
await jisho.searchForPhrase('日').then((result) { await jisho.searchForPhrase('日').then((result) {

12
example/parser/parse_example_page.dart
View File

@ -1,15 +1,15 @@
import 'package:http/http.dart' as http;
import 'dart:convert'; import 'dart:convert';
import 'package:http/http.dart' as http;
import 'package:unofficial_jisho_api/parser.dart' as jisho_parser; import 'package:unofficial_jisho_api/parser.dart' as jisho_parser;
final encoder = JsonEncoder.withIndent(' '); final JsonEncoder encoder = JsonEncoder.withIndent(' ');
const SEARCH_EXAMPLE = '保護者'; const String searchExample = '保護者';
final SEARCH_URI = jisho_parser.uriForExampleSearch(SEARCH_EXAMPLE); final String searchURI = jisho_parser.uriForExampleSearch(searchExample);
void main() async { void main() async {
await http.get(SEARCH_URI).then((result) { await http.get(searchURI).then((result) {
final parsedResult = jisho_parser.parseExamplePageData(result.body, SEARCH_EXAMPLE); final parsedResult = jisho_parser.parseExamplePageData(result.body, searchExample);
print('English: ${parsedResult.results[0].english}'); print('English: ${parsedResult.results[0].english}');
print('Kanji ${parsedResult.results[0].kanji}'); print('Kanji ${parsedResult.results[0].kanji}');
print('Kana: ${parsedResult.results[0].kana}'); print('Kana: ${parsedResult.results[0].kana}');

12
example/parser/parse_kanji_page.dart
View File

@ -1,15 +1,15 @@
import 'package:http/http.dart' as http;
import 'dart:convert'; import 'dart:convert';
import 'package:http/http.dart' as http;
import 'package:unofficial_jisho_api/parser.dart' as jisho_parser; import 'package:unofficial_jisho_api/parser.dart' as jisho_parser;
final encoder = JsonEncoder.withIndent(' '); final JsonEncoder encoder = JsonEncoder.withIndent(' ');
const SEARCH_KANJI = '車'; const String searchKanji = '車';
final SEARCH_URI = jisho_parser.uriForKanjiSearch(SEARCH_KANJI); final String searchURI = jisho_parser.uriForKanjiSearch(searchKanji);
void main() async { void main() async {
await http.get(SEARCH_URI).then((result) { await http.get(searchURI).then((result) {
final parsedResult = jisho_parser.parseKanjiPageData(result.body, SEARCH_KANJI); final parsedResult = jisho_parser.parseKanjiPageData(result.body, searchKanji);
print('JLPT level: ${parsedResult.jlptLevel}'); print('JLPT level: ${parsedResult.jlptLevel}');
print('Stroke count: ${parsedResult.strokeCount}'); print('Stroke count: ${parsedResult.strokeCount}');
print('Meaning: ${parsedResult.meaning}'); print('Meaning: ${parsedResult.meaning}');

12
example/parser/parse_phrase_page.dart
View File

@ -1,16 +1,16 @@
import 'package:http/http.dart' as http;
import 'dart:convert'; import 'dart:convert';
import 'package:http/http.dart' as http;
import 'package:unofficial_jisho_api/parser.dart' as jisho_parser; import 'package:unofficial_jisho_api/parser.dart' as jisho_parser;
final encoder = JsonEncoder.withIndent(' '); final JsonEncoder encoder = JsonEncoder.withIndent(' ');
const SEARCH_EXAMPLE = '保護者'; const String searchExample = '保護者';
final SEARCH_URI = jisho_parser.uriForPhraseScrape(SEARCH_EXAMPLE); final String searchURI = jisho_parser.uriForPhraseScrape(searchExample);
void main() async { void main() async {
await http.get(SEARCH_URI).then((result) { await http.get(searchURI).then((result) {
final parsedResult = jisho_parser.parsePhrasePageData(result.body, SEARCH_EXAMPLE); final parsedResult = jisho_parser.parsePhrasePageData(result.body, searchExample);
print(encoder.convert(parsedResult)); print(encoder.convert(parsedResult));
}); });
} }

10
lib/api.dart
View File

@ -3,14 +3,14 @@
/// It provides a built-in http client and performs async requests to the server /// It provides a built-in http client and performs async requests to the server
/// for different types of requests. /// for different types of requests.
library unofficial_jisho_api; library unofficial_jisho_api;
import './src/objects.dart';
import 'package:http/http.dart' as http;
import 'dart:convert'; import 'dart:convert';
import 'package:http/http.dart' as http;
import './src/phraseSearch.dart';
import './src/kanjiSearch.dart';
import './src/exampleSearch.dart'; import './src/exampleSearch.dart';
import './src/kanjiSearch.dart';
import './src/objects.dart';
import './src/phraseScrape.dart'; import './src/phraseScrape.dart';
import './src/phraseSearch.dart';
/// Query the official Jisho API for a word or phrase /// Query the official Jisho API for a word or phrase
/// ///
@ -53,6 +53,6 @@ Future<PhrasePageScrapeResult> scrapeForPhrase(String phrase) async {
// ); // );
// } // }
throw err; rethrow;
} }
} }

6
lib/parser.dart
View File

@ -5,7 +5,7 @@
/// for providing HTML. /// for providing HTML.
library unofficial_jisho_parser; library unofficial_jisho_parser;
export './src/phraseSearch.dart';
export './src/kanjiSearch.dart' show uriForKanjiSearch, parseKanjiPageData;
export './src/exampleSearch.dart' show uriForExampleSearch, parseExamplePageData; export './src/exampleSearch.dart' show uriForExampleSearch, parseExamplePageData;
export './src/phraseScrape.dart' show uriForPhraseScrape, parsePhrasePageData; export './src/kanjiSearch.dart' show uriForKanjiSearch, parseKanjiPageData;
export './src/phraseScrape.dart' show uriForPhraseScrape, parsePhrasePageData;
export './src/phraseSearch.dart';

28
lib/src/exampleSearch.dart
View File

@ -1,17 +1,17 @@
import './baseURI.dart';
import './objects.dart';
import 'package:html/parser.dart'; import 'package:html/parser.dart';
import 'package:html/dom.dart'; import 'package:html/dom.dart';
final RegExp kanjiRegex = RegExp(r'[\u4e00-\u9faf\u3400-\u4dbf]'); import './baseURI.dart';
import './objects.dart';
final RegExp _kanjiRegex = RegExp(r'[\u4e00-\u9faf\u3400-\u4dbf]');
/// Provides the URI for an example search /// Provides the URI for an example search
String uriForExampleSearch(String phrase) { String uriForExampleSearch(String phrase) {
return '${SCRAPE_BASE_URI}${Uri.encodeComponent(phrase)}%23sentences'; return '$SCRAPE_BASE_URI${Uri.encodeComponent(phrase)}%23sentences';
} }
List<Element> getChildrenAndSymbols(Element ul) { List<Element> _getChildrenAndSymbols(Element ul) {
final ulText = ul.text; final ulText = ul.text;
final ulCharArray = ulText.split(''); final ulCharArray = ulText.split('');
final ulChildren = ul.children; final ulChildren = ul.children;
@ -25,7 +25,7 @@ List<Element> getChildrenAndSymbols(Element ul) {
symbols += ulCharArray[offsetPointer]; symbols += ulCharArray[offsetPointer];
offsetPointer++; offsetPointer++;
} }
final symbolElement = Element.html('<span>${symbols}</span>'); final symbolElement = Element.html('<span>$symbols</span>');
result.add(symbolElement); result.add(symbolElement);
} }
offsetPointer += element.text.length; offsetPointer += element.text.length;
@ -33,15 +33,15 @@ List<Element> getChildrenAndSymbols(Element ul) {
} }
if (offsetPointer + 1 != ulText.length){ if (offsetPointer + 1 != ulText.length){
final symbols = ulText.substring(offsetPointer, ulText.length-1); final symbols = ulText.substring(offsetPointer, ulText.length-1);
final symbolElement = Element.html('<span>${symbols}</span>'); final symbolElement = Element.html('<span>$symbols</span>');
result.add(symbolElement); result.add(symbolElement);
} }
return result; return result;
} }
ExampleResultData getKanjiAndKana(Element div) { ExampleResultData _getKanjiAndKana(Element div) {
final ul = div.querySelector('ul'); final ul = div.querySelector('ul');
final contents = getChildrenAndSymbols(ul); final contents = _getChildrenAndSymbols(ul);
var kanji = ''; var kanji = '';
var kana = ''; var kana = '';
@ -59,7 +59,7 @@ ExampleResultData getKanjiAndKana(Element div) {
final kanaEnding = []; final kanaEnding = [];
for (var j = unlifted.length - 1; j > 0; j -= 1) { for (var j = unlifted.length - 1; j > 0; j -= 1) {
final char = unlifted[j]; final char = unlifted[j];
if (!kanjiRegex.hasMatch(char)) { if (!_kanjiRegex.hasMatch(char)) {
kanaEnding.add(char); kanaEnding.add(char);
} else { } else {
break; break;
@ -100,8 +100,8 @@ List<ExampleSentencePiece> getPieces(Element sentenceElement) {
return pieces; return pieces;
} }
ExampleResultData parseExampleDiv(Element div) { ExampleResultData _parseExampleDiv(Element div) {
final result = getKanjiAndKana(div); final result = _getKanjiAndKana(div);
result.english = div.querySelector('.english').text; result.english = div.querySelector('.english').text;
result.pieces = getPieces(div) ?? []; result.pieces = getPieces(div) ?? [];
@ -113,7 +113,7 @@ ExampleResults parseExamplePageData(String pageHtml, String phrase) {
final document = parse(pageHtml); final document = parse(pageHtml);
final divs = document.querySelectorAll('.sentence_content'); final divs = document.querySelectorAll('.sentence_content');
final results = divs.map((div) => parseExampleDiv(div)).toList(); final results = divs.map((div) => _parseExampleDiv(div)).toList();
return ExampleResults( return ExampleResults(
query: phrase, query: phrase,

128
lib/src/kanjiSearch.dart
View File

@ -1,49 +1,49 @@
import 'package:html_unescape/html_unescape.dart' as html_entities;
import './baseURI.dart'; import './baseURI.dart';
import './objects.dart'; import './objects.dart';
final _htmlUnescape = html_entities.HtmlUnescape();
import 'package:html_unescape/html_unescape.dart' as html_entities; const _onyomiLocatorSymbol = 'On';
final htmlUnescape = html_entities.HtmlUnescape(); const _kunyomiLocatorSymbol = 'Kun';
const String ONYOMI_LOCATOR_SYMBOL = 'On'; String _removeNewlines(String str) {
const String KUNYOMI_LOCATOR_SYMBOL = 'Kun';
String removeNewlines(String str) {
return str.replaceAll(RegExp(r'(?:\r|\n)') , '').trim(); return str.replaceAll(RegExp(r'(?:\r|\n)') , '').trim();
} }
/// Provides the URI for a kanji search /// Provides the URI for a kanji search
String uriForKanjiSearch(String kanji) { String uriForKanjiSearch(String kanji) {
return '${SCRAPE_BASE_URI}${Uri.encodeComponent(kanji)}%23kanji'; return '$SCRAPE_BASE_URI${Uri.encodeComponent(kanji)}%23kanji';
} }
String getUriForStrokeOrderDiagram(String kanji) { String _getUriForStrokeOrderDiagram(String kanji) {
return '${STROKE_ORDER_DIAGRAM_BASE_URI}${kanji.codeUnitAt(0)}_frames.png'; return '$STROKE_ORDER_DIAGRAM_BASE_URI${kanji.codeUnitAt(0)}_frames.png';
} }
bool containsKanjiGlyph(String pageHtml, String kanji) { bool _containsKanjiGlyph(String pageHtml, String kanji) {
final kanjiGlyphToken = '<h1 class="character" data-area-name="print" lang="ja">${kanji}</h1>'; final kanjiGlyphToken = '<h1 class="character" data-area-name="print" lang="ja">$kanji</h1>';
return pageHtml.contains(kanjiGlyphToken); return pageHtml.contains(kanjiGlyphToken);
} }
String getStringBetweenIndicies(String data, int startIndex, int endIndex) { String _getStringBetweenIndicies(String data, int startIndex, int endIndex) {
final result = data.substring(startIndex, endIndex); final result = data.substring(startIndex, endIndex);
return removeNewlines(result).trim(); return _removeNewlines(result).trim();
} }
String getStringBetweenStrings(String data, String startString, String endString) { String _getStringBetweenStrings(String data, String startString, String endString) {
final regex = RegExp('${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}', dotAll: true); final regex = RegExp('${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}', dotAll: true);
final match = regex.allMatches(data).toList(); final match = regex.allMatches(data).toList();
return match.isNotEmpty ? match[0].group(1).toString() : null; return match.isNotEmpty ? match[0].group(1).toString() : null;
} }
int getIntBetweenStrings(String pageHtml, String startString, String endString) { int _getIntBetweenStrings(String pageHtml, String startString, String endString) {
final stringBetweenStrings = getStringBetweenStrings(pageHtml, startString, endString); final stringBetweenStrings = _getStringBetweenStrings(pageHtml, startString, endString);
return int.parse(stringBetweenStrings); return int.parse(stringBetweenStrings);
} }
List<String> getAllGlobalGroupMatches(String str, RegExp regex) { List<String> _getAllGlobalGroupMatches(String str, RegExp regex) {
var regexResults = regex.allMatches(str).toList(); var regexResults = regex.allMatches(str).toList();
List<String> results = []; List<String> results = [];
for (var match in regexResults) { for (var match in regexResults) {
@ -53,59 +53,59 @@ List<String> getAllGlobalGroupMatches(String str, RegExp regex) {
return results; return results;
} }
List<String> parseAnchorsToArray(String str) { List<String> _parseAnchorsToArray(String str) {
final regex = RegExp(r'<a href=".*?">(.*?)<\/a>'); final regex = RegExp(r'<a href=".*?">(.*?)<\/a>');
return getAllGlobalGroupMatches(str, regex); return _getAllGlobalGroupMatches(str, regex);
} }
List<String> getYomi(String pageHtml, String yomiLocatorSymbol) { List<String> _getYomi(String pageHtml, String yomiLocatorSymbol) {
final yomiSection = getStringBetweenStrings(pageHtml, '<dt>${yomiLocatorSymbol}:</dt>', '</dl>'); final yomiSection = _getStringBetweenStrings(pageHtml, '<dt>$yomiLocatorSymbol:</dt>', '</dl>');
return parseAnchorsToArray(yomiSection ?? ''); return _parseAnchorsToArray(yomiSection ?? '');
} }
List<String> getKunyomi(String pageHtml) { List<String> _getKunyomi(String pageHtml) {
return getYomi(pageHtml, KUNYOMI_LOCATOR_SYMBOL); return _getYomi(pageHtml, _kunyomiLocatorSymbol);
} }
List<String> getOnyomi(String pageHtml) { List<String> _getOnyomi(String pageHtml) {
return getYomi(pageHtml, ONYOMI_LOCATOR_SYMBOL); return _getYomi(pageHtml, _onyomiLocatorSymbol);
} }
List<YomiExample> getYomiExamples(String pageHtml, String yomiLocatorSymbol) { List<YomiExample> _getYomiExamples(String pageHtml, String yomiLocatorSymbol) {
final locatorString = '<h2>${yomiLocatorSymbol} reading compounds</h2>'; final locatorString = '<h2>$yomiLocatorSymbol reading compounds</h2>';
final exampleSection = getStringBetweenStrings(pageHtml, locatorString, '</ul>'); final exampleSection = _getStringBetweenStrings(pageHtml, locatorString, '</ul>');
if (exampleSection==null) { if (exampleSection==null) {
return null; return null;
} }
final regex = RegExp(r'<li>(.*?)<\/li>', dotAll: true); final regex = RegExp(r'<li>(.*?)<\/li>', dotAll: true);
final regexResults = getAllGlobalGroupMatches(exampleSection, regex).map((s) => s.trim()); final regexResults = _getAllGlobalGroupMatches(exampleSection, regex).map((s) => s.trim());
final examples = regexResults.map((regexResult) { final examples = regexResults.map((regexResult) {
final examplesLines = regexResult.split('\n').map((s) => s.trim()).toList(); final examplesLines = regexResult.split('\n').map((s) => s.trim()).toList();
return YomiExample( return YomiExample(
example: examplesLines[0], example: examplesLines[0],
reading: examplesLines[1].replaceAll('【', '').replaceAll('】', ''), reading: examplesLines[1].replaceAll('【', '').replaceAll('】', ''),
meaning: htmlUnescape.convert(examplesLines[2]), meaning: _htmlUnescape.convert(examplesLines[2]),
); );
}); });
return examples.toList(); return examples.toList();
} }
List<YomiExample> getOnyomiExamples(String pageHtml) { List<YomiExample> _getOnyomiExamples(String pageHtml) {
return getYomiExamples(pageHtml, ONYOMI_LOCATOR_SYMBOL); return _getYomiExamples(pageHtml, _onyomiLocatorSymbol);
} }
List<YomiExample> getKunyomiExamples(String pageHtml) { List<YomiExample> _getKunyomiExamples(String pageHtml) {
return getYomiExamples(pageHtml, KUNYOMI_LOCATOR_SYMBOL); return _getYomiExamples(pageHtml, _kunyomiLocatorSymbol);
} }
Radical getRadical(String pageHtml) { Radical _getRadical(String pageHtml) {
const radicalMeaningStartString = '<span class="radical_meaning">'; const radicalMeaningStartString = '<span class="radical_meaning">';
const radicalMeaningEndString = '</span>'; const radicalMeaningEndString = '</span>';
var radicalMeaning = getStringBetweenStrings( var radicalMeaning = _getStringBetweenStrings(
pageHtml, pageHtml,
radicalMeaningStartString, radicalMeaningStartString,
radicalMeaningEndString, radicalMeaningEndString,
@ -123,7 +123,7 @@ Radical getRadical(String pageHtml) {
const radicalSymbolEndString = '</span>'; const radicalSymbolEndString = '</span>';
final radicalSymbolEndIndex = pageHtml.indexOf(radicalSymbolEndString, radicalSymbolStartIndex); final radicalSymbolEndIndex = pageHtml.indexOf(radicalSymbolEndString, radicalSymbolStartIndex);
final radicalSymbolsString = getStringBetweenIndicies( final radicalSymbolsString = _getStringBetweenIndicies(
pageHtml, pageHtml,
radicalSymbolStartIndex, radicalSymbolStartIndex,
radicalSymbolEndIndex, radicalSymbolEndIndex,
@ -153,64 +153,64 @@ Radical getRadical(String pageHtml) {
return null; return null;
} }
List<String> getParts(String pageHtml) { List<String> _getParts(String pageHtml) {
const partsSectionStartString = '<dt>Parts:</dt>'; const partsSectionStartString = '<dt>Parts:</dt>';
const partsSectionEndString = '</dl>'; const partsSectionEndString = '</dl>';
final partsSection = getStringBetweenStrings( final partsSection = _getStringBetweenStrings(
pageHtml, pageHtml,
partsSectionStartString, partsSectionStartString,
partsSectionEndString, partsSectionEndString,
); );
var result = parseAnchorsToArray(partsSection); var result = _parseAnchorsToArray(partsSection);
result.sort(); result.sort();
return (result); return (result);
} }
String getSvgUri(String pageHtml) { String _getSvgUri(String pageHtml) {
var svgRegex = RegExp('\/\/.*?.cloudfront.net\/.*?.svg'); var svgRegex = RegExp('\/\/.*?.cloudfront.net\/.*?.svg');
final regexResult = svgRegex.firstMatch(pageHtml).group(0).toString(); final regexResult = svgRegex.firstMatch(pageHtml).group(0).toString();
return regexResult.isNotEmpty ? 'https:${regexResult}' : null; return regexResult.isNotEmpty ? 'https:$regexResult' : null;
} }
String getGifUri(String kanji) { String _getGifUri(String kanji) {
final unicodeString = kanji.codeUnitAt(0).toRadixString(16); final unicodeString = kanji.codeUnitAt(0).toRadixString(16);
final fileName = '${unicodeString}.gif'; final fileName = '$unicodeString.gif';
final animationUri = 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/${fileName}'; final animationUri = 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/$fileName';
return animationUri; return animationUri;
} }
int getNewspaperFrequencyRank(String pageHtml) { int _getNewspaperFrequencyRank(String pageHtml) {
final frequencySection = getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>'); final frequencySection = _getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>');
return (frequencySection != null) ? int.parse(getStringBetweenStrings(frequencySection, '<strong>', '</strong>')) : null; return (frequencySection != null) ? int.parse(_getStringBetweenStrings(frequencySection, '<strong>', '</strong>')) : null;
} }
/// Parses a jisho kanji search page to an object /// Parses a jisho kanji search page to an object
KanjiResult parseKanjiPageData(String pageHtml, String kanji) { KanjiResult parseKanjiPageData(String pageHtml, String kanji) {
final result = KanjiResult(); final result = KanjiResult();
result.query = kanji; result.query = kanji;
result.found = containsKanjiGlyph(pageHtml, kanji); result.found = _containsKanjiGlyph(pageHtml, kanji);
if (result.found==false) { if (result.found==false) {
return result; return result;
} }
result.taughtIn = getStringBetweenStrings(pageHtml, 'taught in <strong>', '</strong>'); result.taughtIn = _getStringBetweenStrings(pageHtml, 'taught in <strong>', '</strong>');
result.jlptLevel = getStringBetweenStrings(pageHtml, 'JLPT level <strong>', '</strong>'); result.jlptLevel = _getStringBetweenStrings(pageHtml, 'JLPT level <strong>', '</strong>');
result.newspaperFrequencyRank = getNewspaperFrequencyRank(pageHtml); result.newspaperFrequencyRank = _getNewspaperFrequencyRank(pageHtml);
result.strokeCount = getIntBetweenStrings(pageHtml, '<strong>', '</strong> strokes'); result.strokeCount = _getIntBetweenStrings(pageHtml, '<strong>', '</strong> strokes');
result.meaning = htmlUnescape.convert(removeNewlines(getStringBetweenStrings(pageHtml, '<div class="kanji-details__main-meanings">', '</div>')).trim()); result.meaning = _htmlUnescape.convert(_removeNewlines(_getStringBetweenStrings(pageHtml, '<div class="kanji-details__main-meanings">', '</div>')).trim());
result.kunyomi = getKunyomi(pageHtml) ?? []; result.kunyomi = _getKunyomi(pageHtml) ?? [];
result.onyomi = getOnyomi(pageHtml) ?? []; result.onyomi = _getOnyomi(pageHtml) ?? [];
result.onyomiExamples = getOnyomiExamples(pageHtml) ?? []; result.onyomiExamples = _getOnyomiExamples(pageHtml) ?? [];
result.kunyomiExamples = getKunyomiExamples(pageHtml) ?? []; result.kunyomiExamples = _getKunyomiExamples(pageHtml) ?? [];
result.radical = getRadical(pageHtml); result.radical = _getRadical(pageHtml);
result.parts = getParts(pageHtml) ?? []; result.parts = _getParts(pageHtml) ?? [];
result.strokeOrderDiagramUri = getUriForStrokeOrderDiagram(kanji); result.strokeOrderDiagramUri = _getUriForStrokeOrderDiagram(kanji);
result.strokeOrderSvgUri = getSvgUri(pageHtml); result.strokeOrderSvgUri = _getSvgUri(pageHtml);
result.strokeOrderGifUri = getGifUri(kanji); result.strokeOrderGifUri = _getGifUri(kanji);
result.uri = uriForKanjiSearch(kanji); result.uri = uriForKanjiSearch(kanji);
return result; return result;
} }

46
lib/src/phraseScrape.dart
View File

@ -1,10 +1,10 @@
import './objects.dart';
import './exampleSearch.dart';
import 'package:html/parser.dart'; import 'package:html/parser.dart';
import 'package:html/dom.dart'; import 'package:html/dom.dart';
List<String> getTags(Document document) { import './exampleSearch.dart' show getPieces;
import './objects.dart';
List<String> _getTags(Document document) {
final List<String> tags = []; final List<String> tags = [];
final tagElements = document.querySelectorAll('.concept_light-tag'); final tagElements = document.querySelectorAll('.concept_light-tag');
@ -16,21 +16,21 @@ List<String> getTags(Document document) {
return tags; return tags;
} }
List<String> getMostRecentWordTypes(Element child) { List<String> _getMostRecentWordTypes(Element child) {
return child.text.split(',').map((s) => s.trim().toLowerCase()).toList(); return child.text.split(',').map((s) => s.trim().toLowerCase()).toList();
} }
List<KanjiKanaPair> getOtherForms(Element child) { List<KanjiKanaPair> _getOtherForms(Element child) {
return child.text.split('、') return child.text.split('、')
.map((s) => s.replaceAll('【', '').replaceAll('】', '').split(' ')) .map((s) => s.replaceAll('【', '').replaceAll('】', '').split(' '))
.map((a) => (KanjiKanaPair( kanji: a[0], kana: (a.length == 2) ? a[1] : null ))).toList(); .map((a) => (KanjiKanaPair( kanji: a[0], kana: (a.length == 2) ? a[1] : null ))).toList();
} }
List<String> getNotes(Element child) => child.text.split('\n'); List<String> _getNotes(Element child) => child.text.split('\n');
String getMeaning(Element child) => child.querySelector('.meaning-meaning').text; String _getMeaning(Element child) => child.querySelector('.meaning-meaning').text;
String getMeaningAbstract(Element child) { String _getMeaningAbstract(Element child) {
final meaningAbstract = child.querySelector('.meaning-abstract'); final meaningAbstract = child.querySelector('.meaning-abstract');
if (meaningAbstract == null) return null; if (meaningAbstract == null) return null;
@ -41,13 +41,13 @@ String getMeaningAbstract(Element child) {
return child.querySelector('.meaning-abstract')?.text; return child.querySelector('.meaning-abstract')?.text;
} }
List<String> getSupplemental(Element child) { List<String> _getSupplemental(Element child) {
final supplemental = child.querySelector('.supplemental_info'); final supplemental = child.querySelector('.supplemental_info');
if (supplemental == null) return []; if (supplemental == null) return [];
return supplemental.text.split(',').map((s) => s.trim()).toList(); return supplemental.text.split(',').map((s) => s.trim()).toList();
} }
List<String> getSeeAlsoTerms(List<String> supplemental) { List<String> _getSeeAlsoTerms(List<String> supplemental) {
if (supplemental == null) return []; if (supplemental == null) return [];
final List<String> seeAlsoTerms = []; final List<String> seeAlsoTerms = [];
@ -61,7 +61,7 @@ List<String> getSeeAlsoTerms(List<String> supplemental) {
return seeAlsoTerms; return seeAlsoTerms;
} }
List<PhraseScrapeSentence> getSentences(Element child) { List<PhraseScrapeSentence> _getSentences(Element child) {
final sentenceElements = child.querySelector('.sentences')?.querySelectorAll('.sentence'); final sentenceElements = child.querySelector('.sentences')?.querySelectorAll('.sentence');
if (sentenceElements == null) return []; if (sentenceElements == null) return [];
@ -91,7 +91,7 @@ List<PhraseScrapeSentence> getSentences(Element child) {
return sentences; return sentences;
} }
PhrasePageScrapeResult getMeaningsOtherFormsAndNotes(Document document) { PhrasePageScrapeResult _getMeaningsOtherFormsAndNotes(Document document) {
final returnValues = PhrasePageScrapeResult( otherForms: [], notes: [] ); final returnValues = PhrasePageScrapeResult( otherForms: [], notes: [] );
final meaningsWrapper = document.querySelector('.meanings-wrapper'); final meaningsWrapper = document.querySelector('.meanings-wrapper');
@ -106,20 +106,20 @@ PhrasePageScrapeResult getMeaningsOtherFormsAndNotes(Document document) {
final child = meaningsChildren[meaningIndex]; final child = meaningsChildren[meaningIndex];
if (child.className.contains('meaning-tags')) { if (child.className.contains('meaning-tags')) {
mostRecentWordTypes = getMostRecentWordTypes(child); mostRecentWordTypes = _getMostRecentWordTypes(child);
} else if (mostRecentWordTypes[0] == 'other forms') { } else if (mostRecentWordTypes[0] == 'other forms') {
returnValues.otherForms = getOtherForms(child); returnValues.otherForms = _getOtherForms(child);
} else if (mostRecentWordTypes[0] == 'notes') { } else if (mostRecentWordTypes[0] == 'notes') {
returnValues.notes = getNotes(child); returnValues.notes = _getNotes(child);
} else { } else {
final meaning = getMeaning(child); final meaning = _getMeaning(child);
final meaningAbstract = getMeaningAbstract(child); final meaningAbstract = _getMeaningAbstract(child);
final supplemental = getSupplemental(child); final supplemental = _getSupplemental(child);
final seeAlsoTerms = getSeeAlsoTerms(supplemental); final seeAlsoTerms = _getSeeAlsoTerms(supplemental);
final sentences = getSentences(child); final sentences = _getSentences(child);
meanings.add(PhraseScrapeMeaning( meanings.add(PhraseScrapeMeaning(
seeAlsoTerms: seeAlsoTerms ?? [], seeAlsoTerms: seeAlsoTerms ?? [],
@ -145,12 +145,12 @@ String uriForPhraseScrape(String searchTerm) {
/// Parses a jisho word search page to an object /// Parses a jisho word search page to an object
PhrasePageScrapeResult parsePhrasePageData(String pageHtml, String query) { PhrasePageScrapeResult parsePhrasePageData(String pageHtml, String query) {
final document = parse(pageHtml); final document = parse(pageHtml);
final result = getMeaningsOtherFormsAndNotes(document); final result = _getMeaningsOtherFormsAndNotes(document);
result.query = query; result.query = query;
if (!result.found) return result; if (!result.found) return result;
result.uri = uriForPhraseScrape(query); result.uri = uriForPhraseScrape(query);
result.tags = getTags(document); result.tags = _getTags(document);
return result; return result;
} }

2
lib/src/phraseSearch.dart
View File

@ -3,5 +3,5 @@ import './baseURI.dart';
/// Provides the URI for a phrase search /// Provides the URI for a phrase search
String uriForPhraseSearch(String phrase) { String uriForPhraseSearch(String phrase) {
return '${JISHO_API}?keyword=${Uri.encodeComponent(phrase)}'; return '$JISHO_API?keyword=${Uri.encodeComponent(phrase)}';
} }

1
pubspec.yaml
View File

@ -15,3 +15,4 @@ dev_dependencies:
pedantic: ^1.8.0 pedantic: ^1.8.0
test: ^1.8.0 test: ^1.8.0
path: ^1.6.0 path: ^1.6.0
effective_dart: ^1.0.0