franta-hg@13: /** franta-hg@13: * Free Telco Dictionary franta-hg@13: * Copyright © 2013 František Kučera (frantovo.cz) franta-hg@13: * franta-hg@13: * This program is free software: you can redistribute it and/or modify franta-hg@13: * it under the terms of the GNU General Public License as published by franta-hg@13: * the Free Software Foundation, either version 3 of the License, or franta-hg@13: * (at your option) any later version. franta-hg@13: * franta-hg@13: * This program is distributed in the hope that it will be useful, franta-hg@13: * but WITHOUT ANY WARRANTY; without even the implied warranty of franta-hg@13: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the franta-hg@13: * GNU General Public License for more details. franta-hg@13: * franta-hg@13: * You should have received a copy of the GNU General Public License franta-hg@13: * along with this program. If not, see . franta-hg@13: */ franta-hg@13: package cz.frantovo.telco.dictionary; franta-hg@13: franta-hg@15: import static cz.frantovo.telco.dictionary.Xmlns.*; franta-hg@15: import static cz.frantovo.telco.dictionary.Functions.*; franta-hg@15: import java.io.BufferedWriter; franta-hg@15: import java.io.ByteArrayOutputStream; franta-hg@15: import java.io.DataOutputStream; franta-hg@15: import java.io.File; franta-hg@15: import java.io.FileOutputStream; franta-hg@15: import java.io.FileWriter; franta-hg@15: import java.io.IOException; franta-hg@15: import java.nio.ByteBuffer; franta-hg@15: import java.nio.charset.Charset; franta-hg@15: import java.util.ArrayList; franta-hg@15: import java.util.List; franta-hg@15: import java.util.logging.Level; franta-hg@15: import java.util.logging.Logger; franta-hg@15: import javax.xml.parsers.DocumentBuilder; franta-hg@15: import javax.xml.parsers.DocumentBuilderFactory; franta-hg@15: import javax.xml.parsers.ParserConfigurationException; franta-hg@15: import javax.xml.transform.Transformer; franta-hg@15: import javax.xml.transform.TransformerConfigurationException; franta-hg@15: import javax.xml.transform.TransformerException; franta-hg@15: import javax.xml.transform.TransformerFactory; franta-hg@15: import javax.xml.transform.dom.DOMSource; franta-hg@15: import javax.xml.transform.stream.StreamResult; franta-hg@15: import javax.xml.transform.stream.StreamSource; franta-hg@15: import javax.xml.xpath.XPath; franta-hg@15: import javax.xml.xpath.XPathConstants; franta-hg@15: import javax.xml.xpath.XPathExpression; franta-hg@15: import javax.xml.xpath.XPathExpressionException; franta-hg@15: import javax.xml.xpath.XPathFactory; franta-hg@15: import org.w3c.dom.Document; franta-hg@15: import org.w3c.dom.Node; franta-hg@15: import org.w3c.dom.NodeList; franta-hg@15: import org.xml.sax.SAXException; franta-hg@15: franta-hg@13: /** franta-hg@15: *

franta-hg@15: * Generates dictionary files in StarDict format from source in our XML format. franta-hg@15: *

franta-hg@15: * franta-hg@15: *

franta-hg@15: * Number format should be: 32-bits unsigned number in network byte order franta-hg@15: *

franta-hg@13: * franta-hg@13: * @author Ing. František Kučera (frantovo.cz) franta-hg@13: */ franta-hg@13: public class Generator { franta-hg@13: franta-hg@15: private static final Logger log = Logger.getLogger(Generator.class.getName()); franta-hg@15: private static final String EMAIL_TOKEN = "ixumhht68"; franta-hg@15: private final DocumentBuilderFactory documentBuilderFactory; franta-hg@15: private final DocumentBuilder documentBuilder; franta-hg@15: private final XPathFactory xpathFactory; franta-hg@15: private final XPath xpath; franta-hg@15: private final TransformerFactory xslFactory; franta-hg@15: private final Transformer xsl; franta-hg@15: private final Charset utf8; franta-hg@15: franta-hg@15: public Generator() throws ParserConfigurationException, TransformerConfigurationException { franta-hg@15: utf8 = Charset.forName("UTF-8"); franta-hg@15: franta-hg@15: documentBuilderFactory = DocumentBuilderFactory.newInstance(); franta-hg@15: documentBuilderFactory.setNamespaceAware(true); franta-hg@15: documentBuilder = documentBuilderFactory.newDocumentBuilder(); franta-hg@15: franta-hg@15: xslFactory = TransformerFactory.newInstance(); franta-hg@15: xsl = xslFactory.newTransformer(new StreamSource("concept.xsl")); franta-hg@15: franta-hg@15: xpathFactory = XPathFactory.newInstance(); franta-hg@15: xpath = xpathFactory.newXPath(); franta-hg@15: xpath.setNamespaceContext(getNamespaceContext()); franta-hg@15: } franta-hg@15: franta-hg@15: private void generate(File folder, String filePrefix) { franta-hg@15: File infoFile = new File(folder, filePrefix + ".ifo"); franta-hg@15: File dictFile = new File(folder, filePrefix + ".dict"); franta-hg@15: File indexFile = new File(folder, filePrefix + ".idx"); franta-hg@15: File synonymFile = new File(folder, filePrefix + ".syn"); franta-hg@15: franta-hg@15: FileOutputStream dictOutputStream = null; franta-hg@15: DataOutputStream synonymOutputStream = null; franta-hg@15: DataOutputStream indexOutputStream = null; franta-hg@15: BufferedWriter infoWriter = null; franta-hg@15: franta-hg@15: try { franta-hg@15: dictOutputStream = new FileOutputStream(dictFile); franta-hg@15: synonymOutputStream = new DataOutputStream(new FileOutputStream(synonymFile)); franta-hg@15: indexOutputStream = new DataOutputStream(new FileOutputStream(indexFile)); franta-hg@15: infoWriter = new BufferedWriter(new FileWriter(infoFile)); franta-hg@15: franta-hg@15: Document sourceDocument = documentBuilder.parse("../../data/dictionary.xml"); franta-hg@15: XPathExpression termsXPath = xpath.compile("d:term/@completeForm|d:term/@abbreviation"); franta-hg@15: // TODO: tags - labels/descriptions franta-hg@15: xsl.setParameter("tags", sourceDocument.getElementsByTagNameNS(DICTIONARY, "tags").item(0)); franta-hg@15: franta-hg@15: /** franta-hg@15: * TODO: sorting franta-hg@15: */ franta-hg@15: long offset = 0; franta-hg@15: long conceptIndex = 0; franta-hg@15: long synonymCount = 0; franta-hg@15: for (Node conceptNode : nodeIterable(sourceDocument.getElementsByTagNameNS(DICTIONARY, "concept"))) { franta-hg@15: ByteArrayOutputStream conceptXhtml = new ByteArrayOutputStream(); franta-hg@15: xsl.transform(new DOMSource(conceptNode), new StreamResult(conceptXhtml)); franta-hg@15: int length = conceptXhtml.size(); franta-hg@15: dictOutputStream.write(conceptXhtml.toByteArray()); franta-hg@15: franta-hg@15: NodeList nameNodes = (NodeList) termsXPath.evaluate(conceptNode, XPathConstants.NODESET); franta-hg@15: List names = new ArrayList<>(); franta-hg@15: franta-hg@15: for (Node nameNode : nodeIterable(nameNodes)) { franta-hg@15: String name = nameNode.getTextContent().trim(); franta-hg@15: if (!name.isEmpty()) { franta-hg@15: names.add(name); franta-hg@15: } franta-hg@15: } franta-hg@15: franta-hg@15: synonymCount = +writeSynonyms(synonymOutputStream, names, conceptIndex); franta-hg@15: writeIndex(indexOutputStream, names.get(0), offset, length); franta-hg@15: franta-hg@15: offset = offset + length; franta-hg@15: conceptIndex++; franta-hg@15: } franta-hg@15: franta-hg@15: indexOutputStream.flush(); franta-hg@15: writeInfo(infoWriter, sourceDocument, conceptIndex + 1, synonymCount, indexFile.length()); franta-hg@15: } catch (SAXException | IOException | TransformerException | XPathExpressionException e) { franta-hg@15: log.log(Level.SEVERE, "unable to generate", e); franta-hg@15: } finally { franta-hg@15: close(dictOutputStream); franta-hg@15: close(synonymOutputStream); franta-hg@15: close(indexOutputStream); franta-hg@15: close(infoWriter); franta-hg@15: } franta-hg@15: } franta-hg@15: franta-hg@15: private void writeIndex(DataOutputStream indexOutputStream, String name, long offset, long length) throws IOException { franta-hg@15: indexOutputStream.write(name.getBytes(utf8)); franta-hg@15: indexOutputStream.write(0); franta-hg@15: indexOutputStream.writeInt((int) offset); // unsigned int 32 franta-hg@15: indexOutputStream.writeInt((int) length); // unsigned int 32 franta-hg@15: } franta-hg@15: franta-hg@15: private int writeSynonyms(DataOutputStream synonymOutputStream, List names, long baseIndex) throws IOException { franta-hg@15: if (names.size() > 1) { franta-hg@15: for (int i = 1; i < names.size(); i++) { franta-hg@15: String name = names.get(i); franta-hg@15: synonymOutputStream.write(name.getBytes(utf8)); franta-hg@15: synonymOutputStream.write(0); franta-hg@15: synonymOutputStream.writeInt((int) baseIndex); // unsigned int 32 franta-hg@15: } franta-hg@15: return names.size() - 1; franta-hg@15: } else { franta-hg@15: return 0; franta-hg@15: } franta-hg@15: } franta-hg@15: franta-hg@15: private void writeInfo(BufferedWriter infoWriter, Document sourceDocument, long wordcount, long synwourdcount, long idxfilesize) throws IOException { franta-hg@15: // TODO: values from document metadata franta-hg@15: infoWriter.write("StarDict's dict ifo file\n"); franta-hg@15: infoWriter.write("version=2.4.2\n"); franta-hg@15: infoWriter.write("bookname=Free Telco Dictionary\n"); franta-hg@15: infoWriter.write("wordcount=" + wordcount + "\n"); franta-hg@15: infoWriter.write("synwordcount=" + synwourdcount + "\n"); franta-hg@15: infoWriter.write("idxfilesize=" + idxfilesize + "\n"); franta-hg@15: infoWriter.write("idxoffsetbits=32\n"); franta-hg@15: infoWriter.write("author=František Kučera\n"); franta-hg@15: infoWriter.write("email=telco-dictionary." + EMAIL_TOKEN + "@" + "frantovo.cz\n"); franta-hg@15: infoWriter.write("website=https://telco.frantovo.cz\n"); franta-hg@15: infoWriter.write("description=A dictionary for telecommunications licensed under GNU FDL\n"); franta-hg@15: infoWriter.write("date=2013.07.09\n"); franta-hg@15: infoWriter.write("sametypesequence=h\n"); franta-hg@15: } franta-hg@15: franta-hg@13: public static void main(String[] args) { franta-hg@15: File outputFolder = new File("../../delivery/free-telco-dictionary"); franta-hg@15: outputFolder.mkdir(); franta-hg@15: franta-hg@15: try { franta-hg@15: Generator g = new Generator(); franta-hg@15: g.generate(outputFolder, "telco"); franta-hg@15: } catch (ParserConfigurationException | TransformerConfigurationException e) { franta-hg@15: log.log(Level.SEVERE, "error during initialization", e); franta-hg@15: } franta-hg@13: } franta-hg@13: }