From ceab1878c87e410e88c8d39809570d09dd569402 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Wed, 4 Nov 2009 12:48:28 +0900 Subject: [PATCH] Fixed scripts and removed useless ones --- createcomponentsfile.py | 86 --------------------------------- createdotfile.py | 102 --------------------------------------- createkradfile.py | 55 --------------------- createradicalsfile.py | 67 ------------------------- createstrokesfile.py | 69 -------------------------- createsvgfiles.py | 52 -------------------- harmonize-svg.py | 2 +- listmissingcomponents.py | 61 ----------------------- releasechanges.py | 15 ------ 9 files changed, 1 insertion(+), 508 deletions(-) delete mode 100755 createcomponentsfile.py delete mode 100755 createdotfile.py delete mode 100755 createkradfile.py delete mode 100755 createradicalsfile.py delete mode 100755 createstrokesfile.py delete mode 100755 createsvgfiles.py delete mode 100755 listmissingcomponents.py delete mode 100755 releasechanges.py diff --git a/createcomponentsfile.py b/createcomponentsfile.py deleted file mode 100755 index ff77c4417..000000000 --- a/createcomponentsfile.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009 Alexandre Courbot -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os, codecs, xml.sax -from kanjivg import * - -def getChildsList(group): - ret = [ child for child in group.childs if isinstance(child, StrokeGr) ] - i = 0 - while i < len(ret): - child = ret[i] - if not child.original and not child.element: - ret[i:i+1] = [ c for c in child.childs if isinstance(c, StrokeGr)] - allElts = False - else: i += 1 - return ret - -def getComponents(group): - toProcess = getChildsList(group) - level = 0 - ret = [ ] - alreadyDone = [ ] - while toProcess: - ret.append(str(level)) - nextLevel = [] - appended = False - for next in toProcess: - elt = next.original - if not elt: elt = next.element - if elt and not elt in alreadyDone: - appended = True - ret.append(elt) - alreadyDone.append(elt) - nextLevel += getChildsList(next) - if not appended: - ret = ret[0:-1] - else: level += 1 - toProcess = nextLevel - return ret - -if __name__ == "__main__": - outFile = codecs.open("kanjiscomponents.txt", "w", "utf-8") - for line in licenseString.split('\n'): - outFile.write("# %s\n" % (line)) - outFile.write("""# -# This file gives, for every kanji, the list of its components. Its format is -# similar to the one of kradfile, with the exception that encoding is utf-8. -# A component is a grapheme that is part of a kanji, without regard about -# whether it is a radical or not. Components are given in their drawing order -# and appear as many times as they are present in the kanji.\n""") - - # Read all kanjis - kanjis = [] - for f in os.listdir("XML"): - # Skip variants - if len(f) > 9: continue - if not f.endswith(".xml"): continue - handler = KanjisHandler() - xml.sax.parse("XML/" + f, handler) - kanjis += handler.kanjis.values() - - kanjis.sort(lambda x,y: cmp(x.id, y.id)) - - for kanji in kanjis: - components = getComponents(kanji.root) - if len(components) > 0: - outList = [] - for c in components: - if c not in outList: outList.append(c) - outFile.write('%s %s\n' % (kanji.midashi, " ".join(outList))) - #outFile.write('%s %s\n' % (kanji.midashi, " ".join(components))) diff --git a/createdotfile.py b/createdotfile.py deleted file mode 100755 index d88f0ede6..000000000 --- a/createdotfile.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009 Alexandre Courbot -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import sys, os, codecs, xml.sax -from kanjivg import * - -def getChildsList(group): - ret = [ child for child in group.childs if isinstance(child, StrokeGr) ] - i = 0 - while i < len(ret): - child = ret[i] - if not child.original and not child.element: - ret[i:i+1] = [ c for c in child.childs if isinstance(c, StrokeGr)] - allElts = False - else: i += 1 - return ret - -def getComponents(group): - toProcess = getChildsList(group) - level = 0 - ret = [ ] - alreadyDone = [ ] - while toProcess: - ret.append(str(level)) - nextLevel = [] - appended = False - for next in toProcess: - elt = next.original - if not elt: elt = next.element - if elt and not elt in alreadyDone: - appended = True - ret.append(elt) - alreadyDone.append(elt) - nextLevel += getChildsList(next) - if not appended: - ret = ret[0:-1] - else: level += 1 - toProcess = nextLevel - return ret - -if __name__ == "__main__": - upKanjis = set() - upKanjis.add(u'白') - upKanjis.add(u'羽') - upKanjis.add(u'竹') - - outFile = codecs.open("kanjisgraph.dot", "w", "utf-8") - outFile.write("digraph kanjis {\n") - - # Read all kanjis - kanjis = [] - for f in os.listdir("XML"): - # Skip variants - if len(f) > 9: continue - if not f.endswith(".xml"): continue - handler = KanjisHandler() - xml.sax.parse("XML/" + f, handler) - kanjis += handler.kanjis.values() - - kanjis.sort(lambda x,y: cmp(x.id, y.id)) - - nodes = set() - graphs = set() - while len(upKanjis) > 0: - nextKanjis = set() - for kanji in kanjis: - components = [ child for child in kanji.root.childs if isinstance(child, StrokeGr) ] - for c in components: - if c.original: left = c.original - else: left = c.element - if kanji.root.original: right = kanji.root.original - else: right = kanji.root.element - if not left or not right: continue - if not left in upKanjis: continue - if ord(left) > 0xffff or ord(right) > 0xffff: continue - nextKanjis.add(right) - nodes.add(left) - nodes.add(right) - graphs.add((left, right)) - upKanjis = nextKanjis - - for node in nodes: - outFile.write('\tnode_%x [label="%s"];\n' % (ord(node), node)) - outFile.write('\n') - for (left, right) in graphs: - outFile.write('\tnode_%x -> node_%x;\n' % (ord(left), ord(right))) - outFile.write("}\n") diff --git a/createkradfile.py b/createkradfile.py deleted file mode 100755 index 9cf6d2696..000000000 --- a/createkradfile.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009 Alexandre Courbot -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os, codecs, xml.sax -from kanjivg import * - -def getComponents(group, isRoot = False): - ret = [] - if not isRoot: - if group.original: ret.append(group.original) - elif group.element: ret.append(group.element) - for child in group.childs: - if isinstance(child, StrokeGr): ret += getComponents(child) - return ret - -if __name__ == "__main__": - outFile = codecs.open("kanjiscomponents.txt", "w", "utf-8") - for line in licenseString.split('\n'): - outFile.write("# %s\n" % (line)) - outFile.write("""# -# This file gives, for every kanji, the list of its components. Its format is -# similar to the one of kradfile, with the exception that encoding is utf-8. -# A component is a grapheme that is part of a kanji, without regard about -# whether it is a radical or not. Components are given in their drawing order -# and appear as many times as they are present in the kanji.\n""") - - # Read all kanjis - handler = KanjisHandler() - xml.sax.parse("kanjivg.xml", handler) - kanjis = handler.kanjis.values() - - kanjis.sort(lambda x,y: cmp(x.id, y.id)) - - for kanji in kanjis: - components = getComponents(kanji.root, True) - if len(components) > 0: - outList = [] - for c in components: - if c not in outList: outList.append(c) - outFile.write('%s %s\n' % (kanji.midashi, " ".join(outList))) diff --git a/createradicalsfile.py b/createradicalsfile.py deleted file mode 100755 index a72308f0a..000000000 --- a/createradicalsfile.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009 Alexandre Courbot -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os, codecs, xml.sax -from kanjivg import * - -def getRadicals(group): - ret = [] - if group.radical: - elt = group.element - if elt: ret = [ elt ] - for child in group.childs: - if isinstance(child, StrokeGr): ret += getRadicals(child) - return ret - -if __name__ == "__main__": - outFile = codecs.open("kanjisradicals.txt", "w", "utf-8") - for line in licenseString.split('\n'): - outFile.write("# %s\n" % (line)) - outFile.write("""# -# This file gives, for every kanji, the list of its radicals. Its format is -# similar to the one of kradfile, with the exception that encoding is utf-8.\n""") - - # Read all kanjis - kanjis = [] - for f in os.listdir("XML"): - # Skip variants - if len(f) > 9: continue - if not f.endswith(".xml"): continue - kanjiDescFile = "XML/" + f - handler = KanjisHandler() - xml.sax.parse(kanjiDescFile, handler) - kanjis += handler.kanjis.values() - - kanjis.sort(lambda x,y: cmp(x.id, y.id)) - - radkanjis = {} - for kanji in kanjis: - radicals = [] - for child in kanji.root.childs: - if isinstance(child, StrokeGr): - radicals += getRadicals(child) - for radical in radicals: - if radkanjis.has_key(radical): - radkanjis[radical].add(kanji.midashi) - else: - s = set() - s.add(kanji.midashi) - radkanjis[radical] = set(s) - for rad in radkanjis.keys(): - if len(rad) > 0: - outFile.write('%s %s\n' % (rad, " ".join(radkanjis[rad]))) diff --git a/createstrokesfile.py b/createstrokesfile.py deleted file mode 100755 index 7ad7c9d20..000000000 --- a/createstrokesfile.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009 Alexandre Courbot -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os, codecs, xml.sax, xml.sax.handler, re -from kanjisvg import * - -class KanjiStrokeHandler(BasicHandler): - def __init__(self): - BasicHandler.__init__(self) - self.kanji = "" - self.depth = 0 - self.active = False - self.pathCpt = 0 - - def handle_start_path(self, attrs): - strokeData = attrs["d"] - strokeData = re.sub("[\n\t ,]+", ",", strokeData) - - self.kanji += strokeData - if self.kanji != "": self.kanji += ";" - - def handle_start_g(self, attrs): - if attrs.has_key("id") and attrs["id"] == "Vektorbild": self.active = True - if self.active: - if len(self.kanji) != 0 and self.kanji[-2:] != '[,': - self.kanji += "[," - self.depth += 1 - - def handle_end_g(self): - if self.active: - self.depth -= 1 - -if __name__ == "__main__": - outFile = codecs.open("kanjisstrokes.txt", "w", "utf-8") - for line in licenseString.split('\n'): - outFile.write("# %s\n" % (line)) - - for f in os.listdir("SVG"): - # Skip variations - if len(f) > 9: continue - if not f.endswith(".svg"): continue - code = int(f[0 : -4], 0x10) - # Only output kanjis - if code < 0x4e00 or code > 0x9fbf: continue - uc = unichr(code) - outFile.write(uc + " ") - - parser = xml.sax.make_parser() - handler = KanjiStrokeHandler() - parser.setContentHandler(handler) - parser.setFeature(xml.sax.handler.feature_external_ges, False) - parser.setFeature(xml.sax.handler.feature_external_pes, False) - parser.parse(os.path.join("SVG", f)) - outFile.write(handler.kanji + "\n") diff --git a/createsvgfiles.py b/createsvgfiles.py deleted file mode 100755 index 81d946989..000000000 --- a/createsvgfiles.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -import os, codecs, xml.sax, datetime -from kanjivg import * - -def createSVG(out, kanji): - out.write('\n') - out.write("\n\n") - out.write(""" - -]> - - - - - - -""") - kanji.toSVG(out) - out.write("\n") - -if __name__ == "__main__": - handler = KanjisHandler() - xml.sax.parse("kanjivg.xml", handler) - kanjis = handler.kanjis.values() - - for kanji in kanjis: - out = codecs.open("generated/SVG/" + str(kanji.id) + ".svg", "w", "utf-8") - createSVG(out, kanji) - diff --git a/harmonize-svg.py b/harmonize-svg.py index 225ce0698..8c0819f2d 100755 --- a/harmonize-svg.py +++ b/harmonize-svg.py @@ -85,5 +85,5 @@ for f in os.listdir("SVG"): document, groups = XMLID(open(os.path.join("SVG", f)).read()) tpp = TemplateParser(open("template.svg").read(), kanji, document, groups) tpp.parse() - out = codecs.open(os.path.join("newSVG", f), "w", "utf-8") + out = codecs.open(os.path.join("SVG", f), "w", "utf-8") out.write(tpp.content) diff --git a/listmissingcomponents.py b/listmissingcomponents.py deleted file mode 100755 index 883942d54..000000000 --- a/listmissingcomponents.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009 Alexandre Courbot -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os, codecs, xml.sax -from kanjivg import * - -# Returns the unicode of a character in a unicode string, taking surrogate pairs into account -def realord(s, pos = 0): - code = ord(s[pos]) - if code >= 0xD800 and code < 0xDC00: - if (len(s) <= pos + 1): - print "realord warning: missing surrogate character" - return 0 - code2 = ord(s[pos + 1]) - if code2 >= 0xDC00 and code < 0xE000: - code = 0x10000 + ((code - 0xD800) * 0x400) + (code2 - 0xDC00) - return code - -def addComponents(strokegr, compSet): - if strokegr.element: compSet.add(strokegr.element) - if strokegr.original: compSet.add(strokegr.original) - for child in strokegr.childs: - if isinstance(child, StrokeGr): - addComponents(child, compSet) - - -if __name__ == "__main__": - # Read all kanjis - handler = KanjisHandler() - xml.sax.parse("kanjivg.xml", handler) - kanjis = handler.kanjis.values() - - kanjis.sort(lambda x,y: cmp(x.id, y.id)) - - componentsList = set() - for kanji in kanjis: - addComponents(kanji.root, componentsList) - print len(componentsList) - - missingComponents = set() - for component in componentsList: - key = hex(realord(component))[2:] - if not handler.kanjis.has_key(key): missingComponents.add(component) - print "Missing components:" - for component in missingComponents: - print component, hex(realord(component)) diff --git a/releasechanges.py b/releasechanges.py deleted file mode 100755 index 6651aea46..000000000 --- a/releasechanges.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -# Load XML and SVG, merge them and list stroke count mismatchs - -# Generate release XML file - -# Generate SVG files with embedded KanjiVG information -# TODO finish embedding all KanjiVG information - - - -# Generate stroke count mismatchs page - -# Generate wiki pages for new kanjis