Split into files

lib/src/baseURI.dart

@@ -0,0 +1,3 @@
+const String JISHO_API = 'https://jisho.org/api/v1/search/words';
+const String SCRAPE_BASE_URI = 'https://jisho.org/search/';
+const String STROKE_ORDER_DIAGRAM_BASE_URI = 'https://classic.jisho.org/static/images/stroke_diagrams/';

lib/src/exampleSearch.dart

@@ -0,0 +1,108 @@
+import './baseURI.dart';
+import './objects.dart';
+
+import 'package:html/parser.dart';
+import 'package:html/dom.dart';
+
+final RegExp kanjiRegex = RegExp(r'[\u4e00-\u9faf\u3400-\u4dbf]');
+
+String uriForExampleSearch(String phrase) {
+  return '${SCRAPE_BASE_URI}${Uri.encodeComponent(phrase)}%23sentences';
+}
+
+/* TODO: This is the wrong approach. 
+ * Symbols such as 、「」。 are missing in mid sentence
+ * Maybe also JP fullwidth numbers?
+ */
+
+String getEndSymbolsOfExampleSentence(Element ul) {
+  final endSymbols = RegExp(r'<\/li>([^<>]+)$');
+  return endSymbols.firstMatch(ul.innerHtml).group(1);
+}
+
+ExampleResultData getKanjiAndKana(Element div) {
+  final ul = div.querySelector('ul');
+  final contents = ul.children;
+
+  var kanji = '';
+  var kana = '';
+  for (var i = 0; i < contents.length; i += 1) {
+    final content = contents[i];
+    if (content.localName == 'li') {
+      final li = content;
+      final furigana = li.querySelector('.furigana')?.text;
+      final unlifted = li.querySelector('.unlinked')?.text;
+
+      if (furigana != null) {
+        kanji += unlifted;
+        kana += furigana;
+
+        final kanaEnding = [];
+        for (var j = unlifted.length - 1; j > 0; j -= 1) {
+          final char = unlifted[j];
+          if (!kanjiRegex.hasMatch(char)) {
+            kanaEnding.add(char);
+          } else {
+            break;
+          }
+        }
+
+        kana += kanaEnding.reversed.join('');
+      } else {
+        kanji += unlifted;
+        kana += unlifted;
+      }
+    } else {
+      final text = content.text.trim();
+      if (text != null) {
+        kanji += text;
+        kana += text;
+      }
+    }
+  }
+    final endSymbols = getEndSymbolsOfExampleSentence(ul).trim();
+    kanji+= endSymbols;
+    kana += endSymbols;
+
+  return ExampleResultData(
+    kanji: kanji,
+    kana: kana,
+  );
+}
+
+List<ExampleSentencePiece> getPieces(Element sentenceElement) {
+  final pieceElements = sentenceElement.querySelectorAll('li.clearfix');
+  final List<ExampleSentencePiece> pieces = [];
+  for (var pieceIndex = 0; pieceIndex < pieceElements.length; pieceIndex += 1) {
+    final pieceElement = pieceElements[pieceIndex];
+    pieces.add(ExampleSentencePiece(
+      lifted: pieceElement.querySelector('.furigana')?.text,
+      unlifted: pieceElement.querySelector('.unlinked')?.text,
+    ));
+  }
+
+  return pieces;
+}
+
+ExampleResultData parseExampleDiv(Element div) {
+  final result = getKanjiAndKana(div);
+  result.english = div.querySelector('.english').text;
+  result.pieces = getPieces(div) ?? [];
+
+  return result;
+}
+
+ExampleResults parseExamplePageData(String pageHtml, String phrase) {
+  final document = parse(pageHtml);
+  final divs = document.querySelectorAll('.sentence_content');
+
+  final results = divs.map((div) => parseExampleDiv(div)).toList();
+
+  return ExampleResults(
+    query: phrase,
+    found: results.isNotEmpty,
+    results: results ?? [],
+    uri: uriForExampleSearch(phrase),
+    phrase: phrase,
+  );
+}

lib/src/kanjiSearch.dart

@@ -0,0 +1,214 @@
+import './baseURI.dart';
+import './objects.dart';
+
+
+import 'package:html_unescape/html_unescape.dart' as html_entities;
+final htmlUnescape = html_entities.HtmlUnescape();
+
+const String ONYOMI_LOCATOR_SYMBOL = 'On';
+const String KUNYOMI_LOCATOR_SYMBOL = 'Kun';
+
+String removeNewlines(String str) {
+  return str.replaceAll(RegExp(r'(?:\r|\n)') , '').trim();
+}
+
+String uriForKanjiSearch(String kanji) {
+  return '${SCRAPE_BASE_URI}${Uri.encodeComponent(kanji)}%23kanji';
+}
+
+String getUriForStrokeOrderDiagram(String kanji) {
+  return '${STROKE_ORDER_DIAGRAM_BASE_URI}${kanji.codeUnitAt(0)}_frames.png';
+}
+
+bool containsKanjiGlyph(String pageHtml, String kanji) {
+  final kanjiGlyphToken = '<h1 class="character" data-area-name="print" lang="ja">${kanji}</h1>';
+  return pageHtml.contains(kanjiGlyphToken);
+}
+
+String getStringBetweenIndicies(String data, int startIndex, int endIndex) {
+  final result = data.substring(startIndex, endIndex);
+  return removeNewlines(result).trim();
+}
+
+String getStringBetweenStrings(String data, String startString, String endString) {
+  final regex = RegExp('${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}', dotAll: true);
+  final match = regex.allMatches(data).toList();
+
+  return match.isNotEmpty ? match[0].group(1).toString() : null;
+}
+
+int getIntBetweenStrings(String pageHtml, String startString, String endString) {
+  final  stringBetweenStrings = getStringBetweenStrings(pageHtml, startString, endString);
+  return int.parse(stringBetweenStrings);
+}
+
+List<String> getAllGlobalGroupMatches(String str, RegExp regex) {
+  var regexResults = regex.allMatches(str).toList();
+  List<String> results = [];
+  for (var match in regexResults) {
+    results.add(match.group(1));
+  }
+
+  return results;
+}
+
+List<String> parseAnchorsToArray(String str) {
+  final regex = RegExp(r'<a href=".*?">(.*?)<\/a>');
+  return getAllGlobalGroupMatches(str, regex);
+}
+
+List<String> getYomi(String pageHtml, String yomiLocatorSymbol) {
+  final yomiSection = getStringBetweenStrings(pageHtml, '<dt>${yomiLocatorSymbol}:</dt>', '</dl>');
+  return parseAnchorsToArray(yomiSection ?? '');
+}
+
+List<String> getKunyomi(String pageHtml) {
+  return getYomi(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
+}
+
+List<String> getOnyomi(String pageHtml) {
+  return getYomi(pageHtml, ONYOMI_LOCATOR_SYMBOL);
+}
+
+List<YomiExample> getYomiExamples(String pageHtml, String yomiLocatorSymbol) {
+  final locatorString = '<h2>${yomiLocatorSymbol} reading compounds</h2>';
+  final exampleSection = getStringBetweenStrings(pageHtml, locatorString, '</ul>');
+  if (exampleSection==null) {
+    return null;
+  }
+
+  final regex = RegExp(r'<li>(.*?)<\/li>', dotAll: true);
+  final regexResults = getAllGlobalGroupMatches(exampleSection, regex).map((s) => s.trim());
+
+  final examples = regexResults.map((regexResult) {
+    final examplesLines = regexResult.split('\n').map((s) => s.trim()).toList();
+    return YomiExample(
+      example: examplesLines[0],
+      reading: examplesLines[1].replaceAll('【', '').replaceAll('】', ''),
+      meaning: htmlUnescape.convert(examplesLines[2]),
+    );
+  });
+
+  return examples.toList();
+}
+
+List<YomiExample> getOnyomiExamples(String pageHtml) {
+  return getYomiExamples(pageHtml, ONYOMI_LOCATOR_SYMBOL);
+}
+
+List<YomiExample> getKunyomiExamples(String pageHtml) {
+  return getYomiExamples(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
+}
+
+Radical getRadical(String pageHtml) {
+  const radicalMeaningStartString = '<span class="radical_meaning">';
+  const radicalMeaningEndString = '</span>';
+
+  var radicalMeaning = getStringBetweenStrings(
+    pageHtml,
+    radicalMeaningStartString,
+    radicalMeaningEndString,
+  ).trim();
+
+  if (radicalMeaning!=null) {
+    final radicalMeaningStartIndex = pageHtml.indexOf(radicalMeaningStartString);
+
+    final radicalMeaningEndIndex = pageHtml.indexOf(
+      radicalMeaningEndString,
+      radicalMeaningStartIndex,
+    );
+
+    final radicalSymbolStartIndex = radicalMeaningEndIndex + radicalMeaningEndString.length;
+    const radicalSymbolEndString = '</span>';
+    final radicalSymbolEndIndex = pageHtml.indexOf(radicalSymbolEndString, radicalSymbolStartIndex);
+
+    final radicalSymbolsString = getStringBetweenIndicies(
+      pageHtml,
+      radicalSymbolStartIndex,
+      radicalSymbolEndIndex,
+    );
+
+    if (radicalSymbolsString.length > 1) {
+      final radicalForms = radicalSymbolsString
+        .substring(1)
+        .replaceAll('(', '')
+        .replaceAll(')', '')
+        .trim()
+        .split(', ');
+
+      return Radical(
+        symbol: radicalSymbolsString[0],
+        forms: radicalForms ?? [],
+        meaning: radicalMeaning
+      );
+    }
+
+    return Radical (
+      symbol: radicalSymbolsString,
+      meaning: radicalMeaning
+    );
+  }
+
+  return null;
+}
+
+List<String> getParts(String pageHtml) {
+  const partsSectionStartString = '<dt>Parts:</dt>';
+  const partsSectionEndString = '</dl>';
+
+  final partsSection = getStringBetweenStrings(
+    pageHtml,
+    partsSectionStartString,
+    partsSectionEndString,
+  );
+
+  var result = parseAnchorsToArray(partsSection);
+  result.sort();
+
+  return (result);
+}
+
+String getSvgUri(String pageHtml) {
+  var svgRegex = RegExp('\/\/.*?.cloudfront.net\/.*?.svg');
+  final regexResult = svgRegex.firstMatch(pageHtml).group(0).toString();
+  return regexResult.isNotEmpty ? 'https:${regexResult}' : null;
+}
+
+String getGifUri(String kanji) {
+  final unicodeString = kanji.codeUnitAt(0).toRadixString(16);
+  final fileName = '${unicodeString}.gif';
+  final animationUri = 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/${fileName}';
+
+  return animationUri;
+}
+
+int getNewspaperFrequencyRank(String pageHtml) {
+  final frequencySection = getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>');
+  return (frequencySection != null) ? int.parse(getStringBetweenStrings(frequencySection, '<strong>', '</strong>')) : null;
+}
+
+KanjiResult parseKanjiPageData(String pageHtml, String kanji) {
+  final result = KanjiResult();
+  result.query = kanji;
+  result.found = containsKanjiGlyph(pageHtml, kanji);
+  if (result.found==false) {
+    return result;
+  }
+
+  result.taughtIn = getStringBetweenStrings(pageHtml, 'taught in <strong>', '</strong>');
+  result.jlptLevel = getStringBetweenStrings(pageHtml, 'JLPT level <strong>', '</strong>');
+  result.newspaperFrequencyRank = getNewspaperFrequencyRank(pageHtml);
+  result.strokeCount = getIntBetweenStrings(pageHtml, '<strong>', '</strong> strokes');
+  result.meaning = htmlUnescape.convert(removeNewlines(getStringBetweenStrings(pageHtml, '<div class="kanji-details__main-meanings">', '</div>')).trim());
+  result.kunyomi = getKunyomi(pageHtml) ?? [];
+  result.onyomi = getOnyomi(pageHtml) ?? [];
+  result.onyomiExamples = getOnyomiExamples(pageHtml) ?? [];
+  result.kunyomiExamples = getKunyomiExamples(pageHtml) ?? [];
+  result.radical = getRadical(pageHtml);
+  result.parts = getParts(pageHtml) ?? [];
+  result.strokeOrderDiagramUri = getUriForStrokeOrderDiagram(kanji);
+  result.strokeOrderSvgUri = getSvgUri(pageHtml);
+  result.strokeOrderGifUri = getGifUri(kanji);
+  result.uri = uriForKanjiSearch(kanji);
+  return result;
+}

lib/src/phraseScrape.dart

@@ -0,0 +1,154 @@
+import './objects.dart';
+import './exampleSearch.dart';
+
+import 'package:html/parser.dart';
+import 'package:html/dom.dart';
+
+List<String> getTags(Document document) {
+  final List<String> tags = [];
+  final tagElements = document.querySelectorAll('.concept_light-tag');
+
+  for (var i = 0; i < tagElements.length; i += 1) {
+    final tagText = tagElements[i].text;
+    tags.add(tagText);
+  }
+
+  return tags;
+}
+
+List<String> getMostRecentWordTypes(Element child) {
+  return child.text.split(',').map((s) => s.trim().toLowerCase()).toList();
+}
+
+List<KanjiKanaPair> getOtherForms(Element child) {
+  return child.text.split('、')
+    .map((s) => s.replaceAll('【', '').replaceAll('】', '').split(' '))
+    .map((a) => (KanjiKanaPair( kanji: a[0], kana: (a.length == 2) ? a[1] : null ))).toList();
+}
+
+List<String> getNotes(Element child) => child.text.split('\n');
+
+String getMeaning(Element child) => child.querySelector('.meaning-meaning').text;
+
+String getMeaningAbstract(Element child) {
+  final meaningAbstract = child.querySelector('.meaning-abstract');
+  if (meaningAbstract == null) return null;
+  
+  for (var element in meaningAbstract.querySelectorAll('a')) {
+    element.remove();
+  }
+
+  return child.querySelector('.meaning-abstract')?.text;
+}
+
+List<String> getSupplemental(Element child) {
+  final supplemental = child.querySelector('.supplemental_info');
+  if (supplemental == null) return [];
+  return supplemental.text.split(',').map((s) => s.trim()).toList();
+}
+
+List<String> getSeeAlsoTerms(List<String> supplemental) {
+  if (supplemental == null) return [];
+
+  final List<String> seeAlsoTerms = [];
+  for (var i = supplemental.length - 1; i >= 0; i -= 1) {
+    final supplementalEntry = supplemental[i];
+    if (supplementalEntry.startsWith('See also')) {
+      seeAlsoTerms.add(supplementalEntry.replaceAll('See also ', ''));
+      supplemental.removeAt(i);
+    }
+  }
+  return seeAlsoTerms;
+}
+
+List<PhraseScrapeSentence> getSentences(Element child) {
+  final sentenceElements = child.querySelector('.sentences')?.querySelectorAll('.sentence');
+  if (sentenceElements == null) return [];
+
+  final List<PhraseScrapeSentence> sentences = [];
+  for (var sentenceIndex = 0; sentenceIndex < (sentenceElements?.length ?? 0); sentenceIndex += 1) {
+    final sentenceElement = sentenceElements[sentenceIndex];
+
+    final english = sentenceElement.querySelector('.english').text;
+    final pieces = getPieces(sentenceElement);
+
+    sentenceElement.querySelector('.english').remove();
+    for (var element in sentenceElement.children[0].children) {
+      element.querySelector('.furigana')?.remove();
+    }
+
+    final japanese = sentenceElement.text;
+
+    sentences.add(
+      PhraseScrapeSentence(
+        english: english,
+        japanese: japanese,
+        pieces: pieces ?? []
+      )
+    );
+  }
+
+  return sentences;
+}
+
+PhrasePageScrapeResult getMeaningsOtherFormsAndNotes(Document document) {
+  final returnValues = PhrasePageScrapeResult( otherForms: [], notes: [] );
+
+  final meaningsWrapper = document.querySelector('.meanings-wrapper');
+  if (meaningsWrapper == null) return PhrasePageScrapeResult(found: false);
+  returnValues.found = true;
+
+  final meaningsChildren = meaningsWrapper.children;
+
+  final List<PhraseScrapeMeaning> meanings = [];
+  var mostRecentWordTypes = [];
+  for (var meaningIndex = 0; meaningIndex < meaningsChildren.length; meaningIndex += 1) {
+    final child = meaningsChildren[meaningIndex];
+
+    if (child.className.contains('meaning-tags')) {
+      mostRecentWordTypes = getMostRecentWordTypes(child);
+
+    } else if (mostRecentWordTypes[0] == 'other forms') {
+      returnValues.otherForms = getOtherForms(child);
+
+    } else if (mostRecentWordTypes[0] == 'notes') {
+      returnValues.notes = getNotes(child);
+
+    } else {
+      final meaning = getMeaning(child);
+      final meaningAbstract = getMeaningAbstract(child);
+      final supplemental = getSupplemental(child);
+      final seeAlsoTerms = getSeeAlsoTerms(supplemental);
+      final sentences = getSentences(child);
+
+      meanings.add(PhraseScrapeMeaning(
+        seeAlsoTerms: seeAlsoTerms ?? [],
+        sentences: sentences ?? [],
+        definition: meaning,
+        supplemental: supplemental ?? [],
+        definitionAbstract: meaningAbstract,
+        tags: mostRecentWordTypes ?? [],
+      ));
+    }
+  }
+
+  returnValues.meanings = meanings;
+
+  return returnValues;
+}
+
+String uriForPhraseScrape(String searchTerm) {
+  return 'https://jisho.org/word/${Uri.encodeComponent(searchTerm)}';
+}
+
+PhrasePageScrapeResult parsePhrasePageData(String pageHtml, String query) {
+  final document = parse(pageHtml);
+  final result = getMeaningsOtherFormsAndNotes(document);
+
+  result.query = query;
+  if (!result.found) return result;
+  result.uri = uriForPhraseScrape(query);
+  result.tags = getTags(document);
+
+  return result;
+}

lib/src/phraseSearch.dart

@@ -0,0 +1,5 @@
+import './baseURI.dart';
+
+String uriForPhraseSearch(String phrase) {
+  return '${JISHO_API}?keyword=${Uri.encodeComponent(phrase)}';
+}

lib/src/unofficial_jisho_api_base.dart

@@ -1,489 +1,12 @@
 import 'package:unofficial_jisho_api/src/objects.dart';
 import 'package:http/http.dart' as http;
-import 'package:html_unescape/html_unescape.dart' as html_entities;
 import 'dart:convert';
-import 'package:html/parser.dart';
-import 'package:html/dom.dart';
 
-final htmlUnescape = html_entities.HtmlUnescape();
+import './phraseSearch.dart';
+import './kanjiSearch.dart';
+import './exampleSearch.dart';
+import './phraseScrape.dart';
 
-const String JISHO_API = 'https://jisho.org/api/v1/search/words';
-const String SCRAPE_BASE_URI = 'https://jisho.org/search/';
-const String STROKE_ORDER_DIAGRAM_BASE_URI = 'https://classic.jisho.org/static/images/stroke_diagrams/';
-
-/* KANJI SEARCH FUNCTIONS START */
-
-const String ONYOMI_LOCATOR_SYMBOL = 'On';
-const KUNYOMI_LOCATOR_SYMBOL = 'Kun';
-
-String removeNewlines(String str) {
-  return str.replaceAll(RegExp(r'(?:\r|\n)') , '').trim();
-}
-
-String uriForKanjiSearch(String kanji) {
-  return '${SCRAPE_BASE_URI}${Uri.encodeComponent(kanji)}%23kanji';
-}
-
-String getUriForStrokeOrderDiagram(String kanji) {
-  return '${STROKE_ORDER_DIAGRAM_BASE_URI}${kanji.codeUnitAt(0)}_frames.png';
-}
-
-String uriForPhraseSearch(String phrase) {
-  return '${JISHO_API}?keyword=${Uri.encodeComponent(phrase)}';
-}
-
-bool containsKanjiGlyph(String pageHtml, String kanji) {
-  final kanjiGlyphToken = '<h1 class="character" data-area-name="print" lang="ja">${kanji}</h1>';
-  return pageHtml.contains(kanjiGlyphToken);
-}
-
-String getStringBetweenIndicies(String data, int startIndex, int endIndex) {
-  final result = data.substring(startIndex, endIndex);
-  return removeNewlines(result).trim();
-}
-
-String getStringBetweenStrings(String data, String startString, String endString) {
-  final regex = RegExp('${RegExp.escape(startString)}(.*?)${RegExp.escape(endString)}', dotAll: true);
-  final match = regex.allMatches(data).toList();
-
-  return match.isNotEmpty ? match[0].group(1).toString() : null;
-}
-
-int getIntBetweenStrings(String pageHtml, String startString, String endString) {
-  final  stringBetweenStrings = getStringBetweenStrings(pageHtml, startString, endString);
-  return int.parse(stringBetweenStrings);
-}
-
-List<String> getAllGlobalGroupMatches(String str, RegExp regex) {
-  var regexResults = regex.allMatches(str).toList();
-  List<String> results = [];
-  for (var match in regexResults) {
-    results.add(match.group(1));
-  }
-
-  return results;
-}
-
-List<String> parseAnchorsToArray(String str) {
-  final regex = RegExp(r'<a href=".*?">(.*?)<\/a>');
-  return getAllGlobalGroupMatches(str, regex);
-}
-
-List<String> getYomi(String pageHtml, String yomiLocatorSymbol) {
-  final yomiSection = getStringBetweenStrings(pageHtml, '<dt>${yomiLocatorSymbol}:</dt>', '</dl>');
-  return parseAnchorsToArray(yomiSection ?? '');
-}
-
-List<String> getKunyomi(String pageHtml) {
-  return getYomi(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
-}
-
-List<String> getOnyomi(String pageHtml) {
-  return getYomi(pageHtml, ONYOMI_LOCATOR_SYMBOL);
-}
-
-List<YomiExample> getYomiExamples(String pageHtml, String yomiLocatorSymbol) {
-  final locatorString = '<h2>${yomiLocatorSymbol} reading compounds</h2>';
-  final exampleSection = getStringBetweenStrings(pageHtml, locatorString, '</ul>');
-  if (exampleSection==null) {
-    return null;
-  }
-
-  final regex = RegExp(r'<li>(.*?)<\/li>', dotAll: true);
-  final regexResults = getAllGlobalGroupMatches(exampleSection, regex).map((s) => s.trim());
-
-  final examples = regexResults.map((regexResult) {
-    final examplesLines = regexResult.split('\n').map((s) => s.trim()).toList();
-    return YomiExample(
-      example: examplesLines[0],
-      reading: examplesLines[1].replaceAll('【', '').replaceAll('】', ''),
-      meaning: htmlUnescape.convert(examplesLines[2]),
-    );
-  });
-
-  return examples.toList();
-}
-
-List<YomiExample> getOnyomiExamples(String pageHtml) {
-  return getYomiExamples(pageHtml, ONYOMI_LOCATOR_SYMBOL);
-}
-
-List<YomiExample> getKunyomiExamples(String pageHtml) {
-  return getYomiExamples(pageHtml, KUNYOMI_LOCATOR_SYMBOL);
-}
-
-Radical getRadical(String pageHtml) {
-  const radicalMeaningStartString = '<span class="radical_meaning">';
-  const radicalMeaningEndString = '</span>';
-
-  var radicalMeaning = getStringBetweenStrings(
-    pageHtml,
-    radicalMeaningStartString,
-    radicalMeaningEndString,
-  ).trim();
-
-  if (radicalMeaning!=null) {
-    final radicalMeaningStartIndex = pageHtml.indexOf(radicalMeaningStartString);
-
-    final radicalMeaningEndIndex = pageHtml.indexOf(
-      radicalMeaningEndString,
-      radicalMeaningStartIndex,
-    );
-
-    final radicalSymbolStartIndex = radicalMeaningEndIndex + radicalMeaningEndString.length;
-    const radicalSymbolEndString = '</span>';
-    final radicalSymbolEndIndex = pageHtml.indexOf(radicalSymbolEndString, radicalSymbolStartIndex);
-
-    final radicalSymbolsString = getStringBetweenIndicies(
-      pageHtml,
-      radicalSymbolStartIndex,
-      radicalSymbolEndIndex,
-    );
-
-    if (radicalSymbolsString.length > 1) {
-      final radicalForms = radicalSymbolsString
-        .substring(1)
-        .replaceAll('(', '')
-        .replaceAll(')', '')
-        .trim()
-        .split(', ');
-
-      return Radical(
-        symbol: radicalSymbolsString[0],
-        forms: radicalForms ?? [],
-        meaning: radicalMeaning
-      );
-    }
-
-    return Radical (
-      symbol: radicalSymbolsString,
-      meaning: radicalMeaning
-    );
-  }
-
-  return null;
-}
-
-List<String> getParts(String pageHtml) {
-  const partsSectionStartString = '<dt>Parts:</dt>';
-  const partsSectionEndString = '</dl>';
-
-  final partsSection = getStringBetweenStrings(
-    pageHtml,
-    partsSectionStartString,
-    partsSectionEndString,
-  );
-
-  var result = parseAnchorsToArray(partsSection);
-  result.sort();
-
-  return (result);
-}
-
-String getSvgUri(String pageHtml) {
-  var svgRegex = RegExp('\/\/.*?.cloudfront.net\/.*?.svg');
-  final regexResult = svgRegex.firstMatch(pageHtml).group(0).toString();
-  return regexResult.isNotEmpty ? 'https:${regexResult}' : null;
-}
-
-String getGifUri(String kanji) {
-  final unicodeString = kanji.codeUnitAt(0).toRadixString(16);
-  final fileName = '${unicodeString}.gif';
-  final animationUri = 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/${fileName}';
-
-  return animationUri;
-}
-
-int getNewspaperFrequencyRank(String pageHtml) {
-  final frequencySection = getStringBetweenStrings(pageHtml, '<div class="frequency">', '</div>');
-  return (frequencySection != null) ? int.parse(getStringBetweenStrings(frequencySection, '<strong>', '</strong>')) : null;
-}
-
-KanjiResult parseKanjiPageData(String pageHtml, String kanji) {
-  final result = KanjiResult();
-  result.query = kanji;
-  result.found = containsKanjiGlyph(pageHtml, kanji);
-  if (result.found==false) {
-    return result;
-  }
-
-  result.taughtIn = getStringBetweenStrings(pageHtml, 'taught in <strong>', '</strong>');
-  result.jlptLevel = getStringBetweenStrings(pageHtml, 'JLPT level <strong>', '</strong>');
-  result.newspaperFrequencyRank = getNewspaperFrequencyRank(pageHtml);
-  result.strokeCount = getIntBetweenStrings(pageHtml, '<strong>', '</strong> strokes');
-  result.meaning = htmlUnescape.convert(removeNewlines(getStringBetweenStrings(pageHtml, '<div class="kanji-details__main-meanings">', '</div>')).trim());
-  result.kunyomi = getKunyomi(pageHtml) ?? [];
-  result.onyomi = getOnyomi(pageHtml) ?? [];
-  result.onyomiExamples = getOnyomiExamples(pageHtml) ?? [];
-  result.kunyomiExamples = getKunyomiExamples(pageHtml) ?? [];
-  result.radical = getRadical(pageHtml);
-  result.parts = getParts(pageHtml) ?? [];
-  result.strokeOrderDiagramUri = getUriForStrokeOrderDiagram(kanji);
-  result.strokeOrderSvgUri = getSvgUri(pageHtml);
-  result.strokeOrderGifUri = getGifUri(kanji);
-  result.uri = uriForKanjiSearch(kanji);
-  return result;
-}
-
-/* KANJI SEARCH FUNCTIONS END */
-
-/* EXAMPLE SEARCH FUNCTIONS START */
-
-final RegExp kanjiRegex = RegExp(r'[\u4e00-\u9faf\u3400-\u4dbf]');
-
-String uriForExampleSearch(String phrase) {
-  return '${SCRAPE_BASE_URI}${Uri.encodeComponent(phrase)}%23sentences';
-}
-
-/* TODO: This is the wrong approach. 
- * Symbols such as 、「」。 are missing in mid sentence
- * Maybe also JP fullwidth numbers?
- */
-
-String getEndSymbolsOfExampleSentence(Element ul) {
-  final endSymbols = RegExp(r'<\/li>([^<>]+)$');
-  return endSymbols.firstMatch(ul.innerHtml).group(1);
-}
-
-ExampleResultData getKanjiAndKana(Element div) {
-  final ul = div.querySelector('ul');
-  final contents = ul.children;
-
-  var kanji = '';
-  var kana = '';
-  for (var i = 0; i < contents.length; i += 1) {
-    final content = contents[i];
-    if (content.localName == 'li') {
-      final li = content;
-      final furigana = li.querySelector('.furigana')?.text;
-      final unlifted = li.querySelector('.unlinked')?.text;
-
-      if (furigana != null) {
-        kanji += unlifted;
-        kana += furigana;
-
-        final kanaEnding = [];
-        for (var j = unlifted.length - 1; j > 0; j -= 1) {
-          final char = unlifted[j];
-          if (!kanjiRegex.hasMatch(char)) {
-            kanaEnding.add(char);
-          } else {
-            break;
-          }
-        }
-
-        kana += kanaEnding.reversed.join('');
-      } else {
-        kanji += unlifted;
-        kana += unlifted;
-      }
-    } else {
-      final text = content.text.trim();
-      if (text != null) {
-        kanji += text;
-        kana += text;
-      }
-    }
-  }
-    final endSymbols = getEndSymbolsOfExampleSentence(ul).trim();
-    kanji+= endSymbols;
-    kana += endSymbols;
-
-  return ExampleResultData(
-    kanji: kanji,
-    kana: kana,
-  );
-}
-
-List<ExampleSentencePiece> getPieces(Element sentenceElement) {
-  final pieceElements = sentenceElement.querySelectorAll('li.clearfix');
-  final List<ExampleSentencePiece> pieces = [];
-  for (var pieceIndex = 0; pieceIndex < pieceElements.length; pieceIndex += 1) {
-    final pieceElement = pieceElements[pieceIndex];
-    pieces.add(ExampleSentencePiece(
-      lifted: pieceElement.querySelector('.furigana')?.text,
-      unlifted: pieceElement.querySelector('.unlinked')?.text,
-    ));
-  }
-
-  return pieces;
-}
-
-ExampleResultData parseExampleDiv(Element div) {
-  final result = getKanjiAndKana(div);
-  result.english = div.querySelector('.english').text;
-  result.pieces = getPieces(div) ?? [];
-
-  return result;
-}
-
-ExampleResults parseExamplePageData(String pageHtml, String phrase) {
-  final document = parse(pageHtml);
-  final divs = document.querySelectorAll('.sentence_content');
-
-  final results = divs.map((div) => parseExampleDiv(div)).toList();
-
-  return ExampleResults(
-    query: phrase,
-    found: results.isNotEmpty,
-    results: results ?? [],
-    uri: uriForExampleSearch(phrase),
-    phrase: phrase,
-  );
-}
-
-/* EXAMPLE SEARCH FUNCTIONS END */
-
-/* PHRASE SCRAPE FUNCTIONS START */
-
-List<String> getTags(Document document) {
-  final List<String> tags = [];
-  final tagElements = document.querySelectorAll('.concept_light-tag');
-
-  for (var i = 0; i < tagElements.length; i += 1) {
-    final tagText = tagElements[i].text;
-    tags.add(tagText);
-  }
-
-  return tags;
-}
-
-List<String> getMostRecentWordTypes(Element child) {
-  return child.text.split(',').map((s) => s.trim().toLowerCase()).toList();
-}
-
-List<KanjiKanaPair> getOtherForms(Element child) {
-  return child.text.split('、')
-    .map((s) => s.replaceAll('【', '').replaceAll('】', '').split(' '))
-    .map((a) => (KanjiKanaPair( kanji: a[0], kana: (a.length == 2) ? a[1] : null ))).toList();
-}
-
-List<String> getNotes(Element child) => child.text.split('\n');
-
-String getMeaning(Element child) => child.querySelector('.meaning-meaning').text;
-
-String getMeaningAbstract(Element child) {
-  final meaningAbstract = child.querySelector('.meaning-abstract');
-  if (meaningAbstract == null) return null;
-  
-  for (var element in meaningAbstract.querySelectorAll('a')) {
-    element.remove();
-  }
-  
-  return child.querySelector('.meaning-abstract')?.text;
-}
-
-List<String> getSupplemental(Element child) {
-  final supplemental = child.querySelector('.supplemental_info');
-  if (supplemental == null) return [];
-  return supplemental.text.split(',').map((s) => s.trim()).toList();
-}
-
-List<String> getSeeAlsoTerms(List<String> supplemental) {
-  if (supplemental == null) return [];
-
-  final List<String> seeAlsoTerms = [];
-  for (var i = supplemental.length - 1; i >= 0; i -= 1) {
-    final supplementalEntry = supplemental[i];
-    if (supplementalEntry.startsWith('See also')) {
-      seeAlsoTerms.add(supplementalEntry.replaceAll('See also ', ''));
-      supplemental.removeAt(i);
-    }
-  }
-  return seeAlsoTerms;
-}
-
-List<PhraseScrapeSentence> getSentences(Element child) {
-  final sentenceElements = child.querySelector('.sentences')?.querySelectorAll('.sentence');
-  if (sentenceElements == null) return [];
-
-  final List<PhraseScrapeSentence> sentences = [];
-  for (var sentenceIndex = 0; sentenceIndex < (sentenceElements?.length ?? 0); sentenceIndex += 1) {
-    final sentenceElement = sentenceElements[sentenceIndex];
-
-    final english = sentenceElement.querySelector('.english').text;
-    final pieces = getPieces(sentenceElement);
-
-    sentenceElement.querySelector('.english').remove();
-    for (var element in sentenceElement.children[0].children) {
-      element.querySelector('.furigana')?.remove();
-    }
-
-    final japanese = sentenceElement.text;
-
-    sentences.add(
-      PhraseScrapeSentence(
-        english: english,
-        japanese: japanese,
-        pieces: pieces ?? []
-      )
-    );
-  }
-
-  return sentences;
-}
-
-PhrasePageScrapeResult getMeaningsOtherFormsAndNotes(Document document) {
-  final returnValues = PhrasePageScrapeResult( otherForms: [], notes: [] );
-
-  final meaningsWrapper = document.querySelector('.meanings-wrapper');
-  if (meaningsWrapper == null) return PhrasePageScrapeResult(found: false);
-  returnValues.found = true;
-
-  final meaningsChildren = meaningsWrapper.children;
-
-  final List<PhraseScrapeMeaning> meanings = [];
-  var mostRecentWordTypes = [];
-  for (var meaningIndex = 0; meaningIndex < meaningsChildren.length; meaningIndex += 1) {
-    final child = meaningsChildren[meaningIndex];
-
-    if (child.className.contains('meaning-tags')) {
-      mostRecentWordTypes = getMostRecentWordTypes(child);
-
-    } else if (mostRecentWordTypes[0] == 'other forms') {
-      returnValues.otherForms = getOtherForms(child);
-
-    } else if (mostRecentWordTypes[0] == 'notes') {
-      returnValues.notes = getNotes(child);
-
-    } else {
-      final meaning = getMeaning(child);
-      final meaningAbstract = getMeaningAbstract(child);
-      final supplemental = getSupplemental(child);
-      final seeAlsoTerms = getSeeAlsoTerms(supplemental);
-      final sentences = getSentences(child);
-
-      meanings.add(PhraseScrapeMeaning(
-        seeAlsoTerms: seeAlsoTerms ?? [],
-        sentences: sentences ?? [],
-        definition: meaning,
-        supplemental: supplemental ?? [],
-        definitionAbstract: meaningAbstract,
-        tags: mostRecentWordTypes ?? [],
-      ));
-    }
-  }
-
-  returnValues.meanings = meanings;
-
-  return returnValues;
-}
-
-String uriForPhraseScrape(String searchTerm) {
-  return 'https://jisho.org/word/${Uri.encodeComponent(searchTerm)}';
-}
-
-PhrasePageScrapeResult parsePhrasePageData(String pageHtml, String query) {
-  final document = parse(pageHtml);
-  final result = getMeaningsOtherFormsAndNotes(document);
-
-  result.query = query;
-  if (!result.found) return result;
-  result.uri = uriForPhraseScrape(query);
-  result.tags = getTags(document);
-
-  return result;
-}
 
 class JishoApi {
 

test/local_function_test_cases.dart

@@ -1,293 +0,0 @@
-import 'package:unofficial_jisho_api/src/objects.dart';
-import 'package:unofficial_jisho_api/unofficial_jisho_api.dart';
-
-import 'package:test/test.dart';
-import 'dart:convert';
-import 'package:http/http.dart' as http;
-
-void test_local_functions() async {
-
-
-/* KANJI SEARCH FUNCTION TESTS START */
-
-  test('removeNewLines', () {
-    final result = removeNewlines('Line \nwith\r\n Newlines and spaces\n');
-    expect(result, 'Line with Newlines and spaces');
-  });
-
-  test('uriForKanjiSearch', () {
-    final result = uriForKanjiSearch('時');
-    expect(result, 'https://jisho.org/search/%E6%99%82%23kanji');
-  });
-
-  test('getUriForStrokeOrderDiagram', () {
-    final result = getUriForStrokeOrderDiagram('時');
-    expect(result, 'https://classic.jisho.org/static/images/stroke_diagrams/26178_frames.png');
-  });
-
-  test('uriForPhraseSearch', () {
-    final result = uriForPhraseSearch('時間');
-    expect(result, 'https://jisho.org/api/v1/search/words?keyword=%E6%99%82%E9%96%93');
-  });
-
-  final kanjiPage = (await http.get('https://jisho.org/search/%E6%99%82%23kanji')).body;
-
-  test('containsKanjiGlyph', () {
-    final result = containsKanjiGlyph(kanjiPage, '時');
-    expect(result, true);
-  });
-
-  test('getStringBetweenIndicies', () {
-    final result = getStringBetweenIndicies('String\n\rwith\nNewlines', 3, 9);
-    expect(result, 'ingw');
-  });
-
-  test('getStringBetweenStrings', () {
-    const data = 'STArT I want this string END';
-    final result = getStringBetweenStrings(data, 'STArT', 'END');
-    expect(result, ' I want this string ');
-  });
-
-  test('getIntBetweenStrings', () {
-    final result = getIntBetweenStrings(kanjiPage, '<strong>', '</strong> strokes');
-    expect(result, 10);
-  });
-
-  test('getAllGlobalGroupMatches', () {
-
-  });
-
-  test('parseAnchorsToArray', () {
-    const htmlCode = 
-    '''
-    <div class="test">
-      <p>
-        <a href="https://test.test">Hello</a>
-      </p>
-      <a href="//xyz">Hi</a>
-      <span>
-        <p>
-          <a href="">How are you doing</a>
-        </p>
-      </span>
-    </div>
-    ''';
-
-    final result = parseAnchorsToArray(htmlCode);
-    expect(result, [
-      'Hello', 'Hi', 'How are you doing']);
-  });
-
-  test('getYomi', () {
-    final result = getYomi(kanjiPage, 'On');
-    expect(result, ['ジ']);
-
-  });
-
-  test('getKunyomi', () {
-    final result = getKunyomi(kanjiPage);
-    expect(result, ['とき', '-どき']);
-  });
-
-  test('getOnyomi', () {
-    final result = getOnyomi(kanjiPage);
-    expect(result, ['ジ']);
-  });
-
-  test('getYomiExamples', () {
-    final result = getYomiExamples(kanjiPage, 'On');
-    expect(
-      json.encode(result),
-      json.encode([
-        YomiExample(
-          example: '時',
-          reading: 'ジ',
-          meaning: '''hour, o'clock, (specified) time, when ..., during ...'''
-        ),
-        YomiExample(
-          example: '時価',
-          reading: 'ジカ',
-          meaning: 'current value, price, market value'
-        ),
-        YomiExample(
-          example: '零時',
-          reading: 'レイジ',
-          meaning: '''twelve o'clock, midnight, noon'''
-        ),
-        YomiExample(
-          example: '平時',
-          reading: 'ヘイジ',
-          meaning: 'peacetime, time of peace, ordinary times, normal times'
-        ),
-      ])
-    );
-  });
-
-  test('getOnyomiExamples', () {
-    final result = getOnyomiExamples(kanjiPage);
-    expect(
-      json.encode(result), 
-      json.encode([
-        YomiExample(
-          example: '時',
-          reading: 'ジ',
-          meaning: '''hour, o'clock, (specified) time, when ..., during ...'''
-        ),
-        YomiExample(
-          example: '時価',
-          reading: 'ジカ',
-          meaning: 'current value, price, market value'
-        ),
-        YomiExample(
-          example: '零時',
-          reading: 'レイジ',
-          meaning: '''twelve o'clock, midnight, noon'''
-        ),
-        YomiExample(
-          example: '平時',
-          reading: 'ヘイジ',
-          meaning: 'peacetime, time of peace, ordinary times, normal times'
-        ),
-      ])
-    );
-  });
-
-  test('getKunyomiExamples', () {
-    final result = getKunyomiExamples(kanjiPage);
-    expect(
-      json.encode(result), 
-      json.encode([
-        YomiExample(
-          example: '時',
-          reading: 'とき',
-          meaning: 'time, hour, moment, occasion, case, chance, opportunity, season, the times, the age, the day, tense'
-        ),
-        YomiExample(
-          example: '時折',
-          reading: 'ときおり',
-          meaning: 'sometimes'
-        ),
-        YomiExample(
-          example: '切り替え時',
-          reading: 'きりかえとき',
-          meaning: 'time to switch over, response time'
-        ),
-        YomiExample(
-          example: '逢魔が時',
-          reading: 'おうまがとき',
-          meaning: '''twilight, time for disasters (similar to 'the witching hour' but not midnight)'''
-        ),
-      ])
-    );
-  });
-
-  test('getRadical', () {
-    final result = getRadical(kanjiPage);
-    expect(
-      json.encode(result),
-      json.encode(Radical(
-        symbol: '日',
-        meaning: 'sun, day'
-      ))
-    );
-  });
-
-  test('getParts', () {
-    final result = getParts(kanjiPage);
-    expect(result, ['土', '寸', '日']);
-  });
-
-  test('getSvgUri', () {
-    final result = getSvgUri(kanjiPage);
-    expect(result, 'https://d1w6u4xc3l95km.cloudfront.net/kanji-2015-03/06642.svg');
-  });
-
-  test('getGifUri', () {
-    final result = getGifUri(kanjiPage);
-    expect(result, 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/3c.gif');
-  });
-
-  test('getNewspaperFrequencyRank', () {
-    final result = getNewspaperFrequencyRank(kanjiPage);
-    expect(result, 16);
-  });
-
-  test('parseKanjiPageData', () {
-    final result = parseKanjiPageData(kanjiPage, '時');
-
-    final expectedResult = KanjiResult();
-    expectedResult.query = '時';
-    expectedResult.found = true;
-    expectedResult.taughtIn = 'grade 2';
-    expectedResult.jlptLevel = 'N5';
-    expectedResult.newspaperFrequencyRank = 16;
-    expectedResult.strokeCount = 10;
-    expectedResult.meaning = 'time, hour';
-    expectedResult.kunyomi = ['とき', '-どき'];
-    expectedResult.onyomi = ['ジ'];
-    expectedResult.onyomiExamples = 
-    [
-      YomiExample(
-        example: '時',
-        reading: 'ジ',
-        meaning: '''hour, o'clock, (specified) time, when ..., during ...'''
-      ),
-      YomiExample(
-        example: '時価',
-        reading: 'ジカ',
-        meaning: 'current value, price, market value'
-      ),
-      YomiExample(
-        example: '零時',
-        reading: 'レイジ',
-        meaning: '''twelve o'clock, midnight, noon'''
-      ),
-      YomiExample(
-        example: '平時',
-        reading: 'ヘイジ',
-        meaning: 'peacetime, time of peace, ordinary times, normal times'
-      ),
-    ];
-    expectedResult.kunyomiExamples = 
-    [
-      YomiExample(
-        example: '時',
-        reading: 'とき',
-        meaning: 'time, hour, moment, occasion, case, chance, opportunity, season, the times, the age, the day, tense'
-      ),
-      YomiExample(
-        example: '時折',
-        reading: 'ときおり',
-        meaning: 'sometimes'
-      ),
-      YomiExample(
-        example: '切り替え時',
-        reading: 'きりかえとき',
-        meaning: 'time to switch over, response time'
-      ),
-      YomiExample(
-        example: '逢魔が時',
-        reading: 'おうまがとき',
-        meaning: '''twilight, time for disasters (similar to 'the witching hour' but not midnight)'''
-      ),
-    ];
-    expectedResult.radical = 
-    Radical(
-      symbol: '日',
-      meaning: 'sun, day'
-    );
-    expectedResult.parts = ['土', '寸', '日'];
-    expectedResult.strokeOrderDiagramUri = 'https://classic.jisho.org/static/images/stroke_diagrams/26178_frames.png';
-    expectedResult.strokeOrderSvgUri = 'https://d1w6u4xc3l95km.cloudfront.net/kanji-2015-03/06642.svg';
-    expectedResult.strokeOrderGifUri = 'https://raw.githubusercontent.com/mistval/kanji_images/master/gifs/6642.gif';
-    expectedResult.uri = 'https://jisho.org/search/%E6%99%82%23kanji';
-
-    expect(
-      json.encode(result),
-      json.encode(expectedResult)
-      );
-  });
-
-  /* KANJI SEARCH FUNCTION TESTS END */
-
-}

test/unofficial_jisho_api_test.dart

@@ -3,7 +3,6 @@ import 'package:path/path.dart' as path;
 import 'dart:convert';
 
 import 'package:unofficial_jisho_api/unofficial_jisho_api.dart';
-import 'local_function_test_cases.dart' show test_local_functions;
 import 'package:test/test.dart';
 
 final jisho = JishoApi();
@@ -26,11 +25,7 @@ void runTestCases(List<String> testCaseFiles, Function apiFunction) async {
 }
 
 void main() async {
-
-  await test_local_functions();
-
   await runTestCases(getFilePaths('kanji_test_cases'), jisho.searchForKanji);
   await runTestCases(getFilePaths('example_test_cases'), jisho.searchForExamples);
   await runTestCases(getFilePaths('phrase_scrape_test_cases'), jisho.scrapeForPhrase);
-
 }