Some debugging

This commit is contained in:
Oystein Kristoffer Tveit 2020-04-21 22:36:26 +02:00
parent 7157bd1d73
commit 9394d7ea2e
4 changed files with 156 additions and 31 deletions

3
.gitignore vendored
View File

@ -9,3 +9,6 @@ build/
# Directory created by dartdoc # Directory created by dartdoc
doc/api/ doc/api/
# File for running and testing independent code
testfile.dart

View File

@ -8,9 +8,9 @@ final htmlUnescape = html_entities.HtmlUnescape();
// TODO: Put public facing types in this file. // TODO: Put public facing types in this file.
const String JISHO_API = 'http://jisho.org/api/v1/search/words'; const String JISHO_API = 'https://jisho.org/api/v1/search/words';
const String SCRAPE_BASE_URI = 'http://jisho.org/search/'; const String SCRAPE_BASE_URI = 'https://jisho.org/search/';
const String STROKE_ORDER_DIAGRAM_BASE_URI = 'http://classic.jisho.org/static/images/stroke_diagrams/'; const String STROKE_ORDER_DIAGRAM_BASE_URI = 'https://classic.jisho.org/static/images/stroke_diagrams/';
/* KANJI SEARCH FUNCTIONS START */ /* KANJI SEARCH FUNCTIONS START */
@ -33,9 +33,9 @@ String uriForPhraseSearch(String phrase) {
return '${JISHO_API}?keyword=${Uri.encodeComponent(phrase)}'; return '${JISHO_API}?keyword=${Uri.encodeComponent(phrase)}';
} }
bool containsKanjiGlyph(pageHtml, String kanji) { bool containsKanjiGlyph(String pageHtml, String kanji) {
final kanjiGlyphToken = '<h1 class="character" data-area-name="print" lang="ja">${kanji}</h1>'; final kanjiGlyphToken = '<h1 class="character" data-area-name="print" lang="ja">${kanji}</h1>';
return pageHtml.indexOf(kanjiGlyphToken) != -1; return pageHtml.contains(kanjiGlyphToken);
} }
String getStringBetweenIndicies(String data, int startIndex, int endIndex) { String getStringBetweenIndicies(String data, int startIndex, int endIndex) {
@ -44,22 +44,22 @@ String getStringBetweenIndicies(String data, int startIndex, int endIndex) {
} }
String getStringBetweenStrings(String data, String startString, String endString) { String getStringBetweenStrings(String data, String startString, String endString) {
final regex = RegExp('${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}'); final regex = RegExp('${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}', dotAll: true);
final match = regex.allMatches(data).toList(); final match = regex.allMatches(data).toList(); //TODO: Something wrong here
return match ?? match[1].toString(); return match.isNotEmpty ? match[0].group(1).toString() : null;
} }
int getIntBetweenStrings(pageHtml, String startString, String endString) { int getIntBetweenStrings(String pageHtml, String startString, String endString) {
final stringBetweenStrings = getStringBetweenStrings(pageHtml, startString, endString); final stringBetweenStrings = getStringBetweenStrings(pageHtml, startString, endString);
return stringBetweenStrings ?? int.parse(stringBetweenStrings); return int.parse(stringBetweenStrings);
} }
List<String> getAllGlobalGroupMatches(String str, RegExp regex) { List<String> getAllGlobalGroupMatches(String str, RegExp regex) {
var regexResults = regex.allMatches(str).toList(); var regexResults = regex.allMatches(str).toList();
var results = []; List<String> results = [];
for (var match in regexResults) { for (var match in regexResults) {
results.add(match.toString().split(' ')[0]); results.add(match.group(1));
} }
return results; return results;
@ -70,20 +70,20 @@ List<String> parseAnchorsToArray(String str) {
return getAllGlobalGroupMatches(str, regex); return getAllGlobalGroupMatches(str, regex);
} }
List<String> getYomi(pageHtml, String yomiLocatorSymbol) { List<String> getYomi(String pageHtml, String yomiLocatorSymbol) {
final yomiSection = getStringBetweenStrings(pageHtml, '<dt>${yomiLocatorSymbol}:</dt>', '</dl>'); final yomiSection = getStringBetweenStrings(pageHtml, '<dt>${yomiLocatorSymbol}:</dt>', '</dl>');
return parseAnchorsToArray(yomiSection ?? ''); return parseAnchorsToArray(yomiSection ?? '');
} }
List<String> getKunyomi(pageHtml) { List<String> getKunyomi(String pageHtml) {
return getYomi(pageHtml, KUNYOMI_LOCATOR_SYMBOL); return getYomi(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
} }
List<String> getOnyomi(pageHtml) { List<String> getOnyomi(String pageHtml) {
return getYomi(pageHtml, ONYOMI_LOCATOR_SYMBOL); return getYomi(pageHtml, ONYOMI_LOCATOR_SYMBOL);
} }
List<YomiExample> getYomiExamples(pageHtml, String yomiLocatorSymbol) { List<YomiExample> getYomiExamples(String pageHtml, String yomiLocatorSymbol) {
final locatorString = '<h2>${yomiLocatorSymbol} reading compounds</h2>'; final locatorString = '<h2>${yomiLocatorSymbol} reading compounds</h2>';
final exampleSection = getStringBetweenStrings(pageHtml, locatorString, '</ul>'); final exampleSection = getStringBetweenStrings(pageHtml, locatorString, '</ul>');
if (exampleSection==null) { if (exampleSection==null) {
@ -102,18 +102,18 @@ List<YomiExample> getYomiExamples(pageHtml, String yomiLocatorSymbol) {
); );
}); });
return examples; return examples.toList();
} }
List<YomiExample> getOnyomiExamples(pageHtml) { List<YomiExample> getOnyomiExamples(String pageHtml) {
return getYomiExamples(pageHtml, ONYOMI_LOCATOR_SYMBOL); return getYomiExamples(pageHtml, ONYOMI_LOCATOR_SYMBOL);
} }
List<YomiExample> getKunyomiExamples(pageHtml) { List<YomiExample> getKunyomiExamples(String pageHtml) {
return getYomiExamples(pageHtml, KUNYOMI_LOCATOR_SYMBOL); return getYomiExamples(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
} }
Radical getRadical(pageHtml) { Radical getRadical(String pageHtml) {
const radicalMeaningStartString = '<span class="radical_meaning">'; const radicalMeaningStartString = '<span class="radical_meaning">';
const radicalMeaningEndString = '</span>'; const radicalMeaningEndString = '</span>';
@ -165,7 +165,7 @@ Radical getRadical(pageHtml) {
return null; return null;
} }
List<String> getParts(pageHtml) { List<String> getParts(String pageHtml) {
const partsSectionStartString = '<dt>Parts:</dt>'; const partsSectionStartString = '<dt>Parts:</dt>';
const partsSectionEndString = '</dl>'; const partsSectionEndString = '</dl>';
@ -181,7 +181,7 @@ List<String> getParts(pageHtml) {
return (result); return (result);
} }
String getSvgUri(pageHtml) { String getSvgUri(String pageHtml) {
var svgRegex = RegExp('\/\/.*?.cloudfront.net\/.*?.svg/'); var svgRegex = RegExp('\/\/.*?.cloudfront.net\/.*?.svg/');
final regexResult = svgRegex.firstMatch(pageHtml).toString(); final regexResult = svgRegex.firstMatch(pageHtml).toString();
return regexResult ?? 'http:${regexResult}'; return regexResult ?? 'http:${regexResult}';
@ -195,12 +195,12 @@ String getGifUri(String kanji) {
return animationUri; return animationUri;
} }
int getNewspaperFrequencyRank(pageHtml) { int getNewspaperFrequencyRank(String pageHtml) {
final frequencySection = getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>'); final frequencySection = getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>');
return frequencySection ?? int.parse(getStringBetweenStrings(frequencySection, '<strong>', '</strong>')); return frequencySection ?? int.parse(getStringBetweenStrings(frequencySection, '<strong>', '</strong>'));
} }
KanjiResult parseKanjiPageData(pageHtml, String kanji) { KanjiResult parseKanjiPageData(String pageHtml, String kanji) {
final result = KanjiResult(); final result = KanjiResult();
result.query = kanji; result.query = kanji;
result.found = containsKanjiGlyph(pageHtml, kanji); result.found = containsKanjiGlyph(pageHtml, kanji);
@ -304,7 +304,7 @@ ExampleResultData parseExampleDiv(div) {
return result; return result;
} }
ExampleResults parseExamplePageData(pageHtml, String phrase) { ExampleResults parseExamplePageData(String pageHtml, String phrase) {
final document = xml.parse(pageHtml); final document = xml.parse(pageHtml);
final divs = document.descendants.where((node) => node.attributes[0].value == 'sentence_content').toList(); final divs = document.descendants.where((node) => node.attributes[0].value == 'sentence_content').toList();
@ -445,7 +445,7 @@ class JishoApi {
/// @async /// @async
searchForPhrase(String phrase) { searchForPhrase(String phrase) {
final uri = uriForPhraseSearch(phrase); final uri = uriForPhraseSearch(phrase);
return http.read(uri).then((response) => jsonDecode(response).data); return http.get(uri).then((response) => jsonDecode(response.body).data);
} }
/// Scrape the word page for a word/phrase. /// Scrape the word page for a word/phrase.
@ -461,8 +461,8 @@ class JishoApi {
Future<PhrasePageScrapeResult> scrapeForPhrase(String phrase) async { Future<PhrasePageScrapeResult> scrapeForPhrase(String phrase) async {
final uri = uriForPhraseScrape(phrase); final uri = uriForPhraseScrape(phrase);
try { try {
final response = await http.read(uri); final response = await http.get(uri);
return parsePhrasePageData(jsonDecode(response).data, phrase); return parsePhrasePageData(response.body, phrase);
} catch (err) { } catch (err) {
if (err.response.status == 404) { if (err.response.status == 404) {
return PhrasePageScrapeResult( return PhrasePageScrapeResult(
@ -481,7 +481,7 @@ class JishoApi {
/// @async /// @async
Future<KanjiResult> searchForKanji(String kanji) { Future<KanjiResult> searchForKanji(String kanji) {
final uri = uriForKanjiSearch(kanji); final uri = uriForKanjiSearch(kanji);
return http.read(uri).then((response) => parseKanjiPageData(jsonDecode(response).data, kanji)); return http.get(uri).then((response) => parseKanjiPageData(response.body, kanji));
} }
/// Scrape Jisho.org for examples. /// Scrape Jisho.org for examples.
@ -490,6 +490,6 @@ class JishoApi {
/// @async /// @async
Future<ExampleResults> searchForExamples(String phrase) { Future<ExampleResults> searchForExamples(String phrase) {
final uri = uriForExampleSearch(phrase); final uri = uriForExampleSearch(phrase);
return http.read(uri).then((response) => parseExamplePageData(jsonDecode(response).data, phrase)); return http.get(uri).then((response) => parseExamplePageData(response.body, phrase));
} }
} }

View File

@ -0,0 +1,119 @@
import 'package:unofficial_jisho_api/unofficial_jisho_api.dart';
import 'package:test/test.dart';
import 'package:http/http.dart' as http;
void test_local_functions() async {
/* KANJI SEARCH FUNCTION TESTS START */
test('removeNewLines', () {
final result = removeNewlines('Line \nwith\r\n Newlines and spaces\n');
expect(result, 'Line with Newlines and spaces');
});
test('uriForKanjiSearch', () {
final result = uriForKanjiSearch('');
expect(result, 'https://jisho.org/search/%E6%99%82%23kanji');
});
test('getUriForStrokeOrderDiagram', () {
final result = getUriForStrokeOrderDiagram('');
expect(result, 'https://classic.jisho.org/static/images/stroke_diagrams/26178_frames.png');
});
test('uriForPhraseSearch', () {
final result = uriForPhraseSearch('時間');
expect(result, 'https://jisho.org/api/v1/search/words?keyword=%E6%99%82%E9%96%93');
});
final kanjiPage = (await http.get('https://jisho.org/search/%E6%99%82%23kanji')).body;
test('containsKanjiGlyph', () {
final result = containsKanjiGlyph(kanjiPage, '');
expect(result, true);
});
test('getStringBetweenIndicies', () {
final result = getStringBetweenIndicies('String\n\rwith\nNewlines', 3, 9);
expect(result, 'ingw');
});
test('getStringBetweenStrings', () {
const data = 'STArT I want this string END';
final result = getStringBetweenStrings(data, 'STArT', 'END');
expect(result, ' I want this string ');
});
test('getIntBetweenStrings', () {
final result = getIntBetweenStrings(kanjiPage, '<strong>', '</strong> strokes');
expect(result, 10);
});
test('getAllGlobalGroupMatches', () {
});
test('parseAnchorsToArray', () {
});
test('getYomi', () {
final result = getYomi(kanjiPage, 'On');
expect(result, ['']);
});
test('getKunyomi', () {
final result = getKunyomi(kanjiPage);
expect(result, ['とき', '-どき']);
});
test('getOnyomi', () {
final result = getOnyomi(kanjiPage);
expect(result, ['']);
});
test('getYomiExamples', () {
final result = getYomiExamples(kanjiPage, 'Kun');
expect(result, ['']); //FIX
});
test('getOnomiExamples', () {
final result = getOnyomiExamples(kanjiPage);
expect(result, ['']); //FIX
});
test('getKunyomiExamples', () {
final result = getKunyomiExamples(kanjiPage);
expect(result, ['']); //FIX
});
test('getRadical', () {
});
test('getParts', () {
});
test('getSvgUri', () {
});
test('getGifUri', () {
});
test('getNewspaperFrequencyRank', () {
});
test('parseKanjiPageData', () {
});
/* KANJI SEARCH FUNCTION TESTS END */
}

View File

@ -3,6 +3,7 @@ import 'package:path/path.dart' as path;
import 'dart:convert'; import 'dart:convert';
import 'package:unofficial_jisho_api/unofficial_jisho_api.dart'; import 'package:unofficial_jisho_api/unofficial_jisho_api.dart';
import 'local_function_test_cases.dart' show test_local_functions;
import 'package:test/test.dart'; import 'package:test/test.dart';
final jisho = JishoApi(); final jisho = JishoApi();
@ -21,7 +22,7 @@ void runTestCases(List<String> testCaseFiles, String apiFunction) async {
switch(apiFunction) { switch(apiFunction) {
case 'searchForKanji': { case 'searchForKanji': {
final result = await jisho.searchForKanji(testCase['query']); final result = await jisho.searchForKanji(testCase['query']);
expect(result, testCase['expectedResult']); expect(result, jsonDecode(testCase['expectedResult']));
break; break;
} }
case 'searchForExamples': { case 'searchForExamples': {
@ -42,6 +43,8 @@ void runTestCases(List<String> testCaseFiles, String apiFunction) async {
void main() async { void main() async {
await test_local_functions();
await runTestCases(getFilePaths('kanji_test_cases'), 'searchForKanji'); await runTestCases(getFilePaths('kanji_test_cases'), 'searchForKanji');
await runTestCases(getFilePaths('example_test_cases'), 'searchForExamples'); await runTestCases(getFilePaths('example_test_cases'), 'searchForExamples');