use system-specific path separators

--HG--
branch : pypackage

.hgignore

@@ -1,2 +1,7 @@
 syntax: glob
 *.pyc
+*.html
+dist/
+build/
+*.egg-info/
+.git/

.hgtags

@@ -3,3 +3,8 @@ a0f3d678b3cad40e1ea2db938e6b47ad8fa32a3d 2.5.5
 b4343a05f5d584cde0656896f4bf2395c805efcb 2.5.7
 172da49a35f62a305bfe06287fc6da74822039b0 2.5.8
 8138b86a919635f85bd7c8b195d1035ff85e231b 2.5.9
+c48f0d1d29ac0dcf8b912fda56702bbbe5431909 2.5.10
+9b2eff55a3aeb3fad6b99ff5b1fd17c2cd6e5f6a 2.6.0
+6c0e2881db60c4640d36864e2f0007c8fb2df1ae 2.6.1
+f93de5b34430e86b9fd9e9012107f91d6f30d116 2.6.2
+b307d3f5b0bff0fb706d3a3f7bd1c03369b7c224 2.6.3

LICENSE.txt

@@ -0,0 +1 @@
+See COPTING.txt

MANIFEST.in

@@ -0,0 +1,19 @@
+# Files to include in the source package - for the case
+# the project directory is are not in a VCS.
+include yml2c
+include yml2proc
+include yml2/*.ysl2
+include yml2/*.yml2
+
+include Makefile
+include *.yhtml2
+include format.css
+include COPYING.txt
+include README.rst
+include README.md
+
+include samples/*
+include vim/syntax/*
+
+exclude .hgtags
+exclude .hgignore

Makefile

@@ -1,20 +1,54 @@
-YML_PATH=
-YML2C=yml2c
+YML_PATH  =
+DEBVER   := 1
+PYTHON   := python3
+PKGVER    = $(shell awk '/^version/ {print $$3}' setup.cfg)
+TWINE    := $(PYTHON) -m twine
+TWINEREP := pypi
+YML2C     = $(PYTHON) yml2c
 
 all: homepage
 
 homepage: index.html features.html yslt.html toolchain.html programming.html hello.html
 
-update: homepage format.css gpl-2.0.txt
-	rsync -avC *.html *.yml2 format.css *.yhtml2 gpl-2.0.txt samples dragon:fdik.org/yml2/
+update: homepage format.css COPYING.txt
+	rsync -avC *.html *.yml2 format.css *.yhtml2 COPYING.txt samples dragon:fdik.org/yml2/
 
 update-all: update yml2c yml2.py pyPEG.py backend.py yml2proc
 	if test -z $(VERSION) ; then echo VERSION not set ; exit 1 ; fi
-	rsync -avC *.py yml2c Makefile yml2proc xml2yml.ysl2 standardlib.ysl2 samples dragon:fdik.org/yml2/
-	ssh dragon bash -c "cd ; cd fdik.org/; tar cvjf yml-$(VERSION).tar.bz2 yml2/{*.py,*.yml2,*.yhtml2,format.css,gpl-2.0.txt,yml2c,Makefile,yml2proc,xml2yml.ysl2,standardlib.ysl2,samples} ; rm yml2.tar.bz2 ; ln -s yml-$(VERSION).tar.bz2 yml2.tar.bz2"
+	rsync -avC *.py yml2 yml2c Makefile yml2proc xml2yml.ysl2 standardlib.ysl2 samples dragon:fdik.org/yml2/
+	ssh dragon bash -c "cd ; cd fdik.org/; tar cvjf yml-$(VERSION).tar.bz2 yml2/{*.py,*.yml2,*.yhtml2,format.css,COPYING.txt,yml2,yml2c,Makefile,yml2proc,xml2yml.ysl2,standardlib.ysl2,samples} ; rm yml2.tar.bz2 ; ln -s yml-$(VERSION).tar.bz2 yml2.tar.bz2"
 
 %.html: %.en.yhtml2 heading.en.yhtml2 homepage.en.yhtml2
 	$(YML2C) $< -o $@
 
+.PHONY: deb
+deb:  YML2_$(PKGVER).orig.tar.gz python-yml2_$(PKGVER)-$(DEBVER)_all.deb
+
+YML2_$(PKGVER).orig.tar.gz:
+	python setup.py sdist
+	mv -f dist/YML2-$(PKGVER).tar.gz YML2_$(PKGVER).orig.tar.gz
+
+python-yml2_$(PKGVER)-$(DEBVER)_all.deb:
+	$(PYTHON) setup.py --command-packages=stdeb.command bdist_deb
+	mv -f deb_dist/python-yml2_$(PKGVER)-$(DEBVER)_all.deb .
+
 clean:
-	rm -f *.html *.pyc *.pyo
+	rm -f *.html *.pyc *.pyo */*.pyc */*.pyo
+	rm -rf *.egg-info
+	rm -f YML2_$(PKGVER).orig.tar.gz
+	rm -f python-yml2_$(PKGVER)-$(DEBVER)_all.deb
+	rm -f dist/YML2-$(PKGVER).tar.gz
+
+sdist-pypi: dist/YML2-$(PKGVER).tar.gz
+	$(TWINE) check $<
+
+upload-pypi: sdist-pypi
+	$(TWINE) upload --repository "$(TWINEREP)" $<
+
+.PHONY: dist/YML2-$(PKGVER).tar.gz
+dist/YML2-$(PKGVER).tar.gz:
+	$(PYTHON) setup.py sdist
+
+install:
+	pip install -e .
+

README.md

@@ -0,0 +1,144 @@
+## What is YML?
+
+Well, it's the idea not to need to define a grammar first when you want to use a Domain Specific Language. For that purpose, YML is being translated into XML. Let's make an example.
+
+Everything which comes close to a C like language, parses without a grammar definition:
+
+This:
+
+    template< class T > T max(T a, T b);
+
+Parses to:
+
+    <?xml version='1.0' encoding='UTF-8'?>
+    <template>
+      <generic>
+        <class/>
+        <T/>
+      </generic>
+      <T>
+        <max>
+          <parm>
+            <T/>
+            <a/>
+          </parm>
+          <parm>
+            <T/>
+            <b/>
+          </parm>
+        </max>
+      </T>
+    </template>
+
+Instead of defining grammars, you test out and play around until the results are matching your needs. If the resulting tree does not fit what you're expecting, change it by patching the grammar with `decl`:
+
+This:
+
+    module A {
+        interface B {
+            attribute long n;
+        };
+    };
+
+Parses to:
+
+    <?xml version='1.0' encoding='UTF-8'?>
+    <module>
+      <A>
+        <interface>
+          <B>
+            <attribute>
+              <long>
+                <n/>
+              </long>
+            </attribute>
+          </B>
+        </interface>
+      </A>
+    </module>
+
+This does not look like what we want. So we tell YML that we have a module name after the module, an interface name after the interface and type and name after the attribute:
+
+This:
+
+    decl module @name;
+    decl interface @name;
+    decl attribute @type @name;
+
+    module A {
+        interface B {
+            attribute long n;
+        };
+    };
+
+Parses to:
+
+    <?xml version='1.0' encoding='UTF-8'?>
+    <module name="A">
+      <interface name="B">
+        <attribute type="long" name="n"/>
+      </interface>
+    </module>
+
+What can I do with YML?
+
+With YML you can:
+
+  * use a C-like DSL without writing a grammar first
+  * generate code out of this DSL using YSLT
+  * generate code out of UML using YSLT on XMI
+  * generate code out of any XML based language like SVG using YSLT
+  * define a wiki like language in just a few lines like YHTML does
+  * replace bad designed and complicated XML languages with simpler C-like ones
+  * ... and much more.
+
+How it works: Replacing angle brackets with some Python
+
+Just writing down what I wanted to have instead of XML for a sample:
+
+    <list name="List of goods">
+        <head>
+            <columTitle>
+                Goods
+            </columnTitle>
+            <columnTitle>
+                Price
+            </columnTitle>
+        </head>
+        <row>
+            <value>
+                Beer
+            </value>
+            <value>
+                20
+            </value>
+        </row>
+        <row>
+            <value>
+                Wine
+            </value>
+            <value>
+                30
+            </value>
+        </row>
+    </list>
+
+Something like that should be more easy, say, like this:
+
+    list "List of goods" {
+        head title "Goods", title "Price";
+        row value "Beer", value 20;
+        row value "Wine", value 30;
+    }
+
+### Y Languages
+
+The latter is what I call an Y language – a language specified in YML. How could this be achieved? Well, what's to do? To have the required information, how to build XML from the script above, we need:
+
+  * the information, that “list of goods” is an attribute named `name`, while `Goods` is the text value of a tag
+  * `title` shout be written out as `columnTitle`
+
+How to do that? Let's invent a simple definition language for that information:
+
+    decl list(name);
+    decl title alias columnTitle;

README.rst

@@ -0,0 +1,10 @@
+************
+What is YML?
+************
+
+Well, it's the idea not to need to define a grammar first when you want to use a Domain Specific Language. For that purpose, YML is being translated into XML. Let's make an example.
+
+Everything which comes close to a C like language, parses without a grammar definition.
+
+Please see README.md for more information.
+

debian/changelog

@@ -0,0 +1,5 @@
+yml2 (5.8-1) unstable; urgency=low
+
+  * source package automatically created by stdeb 0.8.5
+
+ -- Volker Birk <vb@pep.foundation>  Tue, 04 Sep 2018 17:29:10 +0200

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,18 @@
+Source: yml2
+Maintainer: Volker Birk <vb@pep.foundation>
+Section: python
+Priority: optional
+Build-Depends: dh-python, python-setuptools (>= 0.6b3), python-all (>= 2.6.6-3), debhelper (>= 9)
+Standards-Version: 3.9.6
+Homepage: https://pep.foundation/dev/repos/yml2
+
+Package: python-yml2
+Architecture: all
+Depends: ${misc:Depends}, ${python:Depends}
+Description: YML 2 compiler
+ .
+ YML2 caters the idea not to need to define a grammar first when you want to
+ use a Domain Specific Language. For that purpose, YML is being translated into
+ XML. Let's make an example.
+
+

debian/rules

@@ -0,0 +1,8 @@
+#!/usr/bin/make -f
+
+# This file was automatically generated by stdeb 0.8.5 at
+# Tue, 04 Sep 2018 17:29:10 +0200
+export PYBUILD_NAME=yml2
+%:
+	dh $@ --with python2 --buildsystem=pybuild
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

debian/watch

@@ -0,0 +1,4 @@
+# please also check http://pypi.debian.net/YML2/watch
+version=3
+opts=uversionmangle=s/(rc|a|b|c)/~$1/ \
+http://pypi.debian.net/YML2/YML2-(.+)\.(?:zip|tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))

features.en.yhtml2

@@ -975,6 +975,7 @@ page "The Features" {
         if the «YML_PATH» environment variable is set to a colon separated list of directories,
         these directories are being searched for the given filename. Otherwise, the local
         directory is searched.
+        The system location for «.yml2» and «.ysl2» files is always searched afterwards.
         >>
 
     p   >>

heading.en.yhtml2

@@ -13,10 +13,12 @@ decl ne is a {
 div id=headline {
     p > YML – Why a Markup Language?!
     div class=small {
-        "YML 2.5.9 of Tu 20 Nov 2018  –  Copyleft 2007-2018, "
+        "YML 2.6.3 of Thu 19 Mar 2020  –  Copyleft 2007-2020, "
         a "http://blog.fdik.org", "Volker Birk";
         " – "
         a "http://fdik.org/yml2.tar.bz2", "Download YML 2";
+        " – "
+        a "https://pep.foundation/dev/repos/yml2/", "Mercurial repository";
     }
 }
 

samples/adresslist.yml2

@@ -0,0 +1,12 @@
+list {
+    address {
+        name "Meier";
+        street "Uhlmannstrasse 42";
+        village "Laupheim";
+    }
+    address {
+        name "Schmidt";
+        street "Haupstrasse 23";
+        village "Bad Waldsee";
+    }
+}

samples/gen.bas

@@ -0,0 +1,6 @@
+110 DATA "Meier", "Uhlmannstrasse 42", "Laupheim"
+120 DATA "Schmidt", "Haupstrasse 23", "Bad Waldsee"
+1000 FOR I=1 TO 2 STEP 1
+1010 READ NAME$, STREET$, VILLAGE$
+1020 PRINT "INSERT INTO address (name, street, village) VALUES ('" + NAME$ + "', '" + STREET$ + "', '" + VILLAGE$ + "');"
+1030 NEXT I

samples/gen_addr_sql.ysl2

@@ -0,0 +1,15 @@
+include yslt.yml2
+
+tstylesheet {
+    template "/list" {
+        apply "address", 0;
+        | 1000 FOR I=1 TO 2 STEP 1
+        | 1010 READ NAME$, STREET$, VILLAGE$
+        | 1020 PRINT "INSERT INTO address (name, street, village) VALUES ('" + NAME$ + "', '" + STREET$ + "', '" + VILLAGE$ + "');"
+        | 1030 NEXT I
+    }
+    template "address" {
+        | «position() * 10 + 100» DATA "«name»", "«street»", "«village»"
+    }
+}
+

samples/sample.yml2

@@ -1,7 +0,0 @@
-include idl.yml2
-
-module blub {
-    interface bla {
-        int f(in long a, out long b);
-    }
-}

setup.cfg

@@ -0,0 +1,22 @@
+[sdist]
+formats=gztar
+
+[metadata]
+name         = yml2
+version      = 2.6.4
+author	     = Volker Birk
+author_email = vb@pep-project.org
+url	     = https://pep.foundation/dev/repos/yml2/
+license	     = GPL-2.0
+license_file = COPYING.txt
+description-file = README.rst
+
+[options]
+packages = yml2
+scripts =
+    yml2c
+    yml2proc
+include_package_data = True
+zip_safe = False
+install_requires = lxml
+

setup.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+from setuptools import setup, find_packages
+
+with open('README.rst', 'r', encoding='utf-8') as f:
+    long_description = f.read()
+
+setup(
+    setup_requires = ["setuptools >= 30.3.0"],  # release 2016-12-06
+    long_description=long_description,
+    entry_points = {
+        'console_scripts': [
+            'yml2c=yml2.yml2c:main',
+            'yml2proc=yml2.yml2proc:main'
+        ],
+    },
+    classifiers=[
+      'Development Status :: 5 - Production/Stable',
+      'Intended Audience :: Developers',
+      'Environment :: Console',
+      'Topic :: Software Development :: Build Tools',
+      'License :: OSI Approved :: GNU General Public License v2 (GPLv2)',
+      'Programming Language :: Python :: 3',
+      'Programming Language :: Python :: 3.4',
+      'Programming Language :: Python :: 3.5',
+      'Programming Language :: Python :: 3.6',
+    ],
+)
+

toolchain.en.yhtml2

@@ -87,6 +87,7 @@ page "The YML Toolchain" {
         If «YML_PATH» is set to a colon separated ¬http://en.wikipedia.org/wiki/Path_(computing) path¬
         list, then the `a href="features#including" code "include";` statement searches these paths.
         If «YML_PATH» is not set, the local directory «.» is searched for files.
+        The system location for «.yml2» and «.ysl2» files is always searched afterwards.
         >>
 
     h2 id=processor > The YML/YSLT processor
@@ -218,8 +219,6 @@ page "The YML Toolchain" {
 
     p > Convert XML to normalized YML. Equivalent to -xy xml2yml.ysl2
 
-    p > The file xml2yml.ysl2 has to be in YML_PATH.
-
     h3 > -V, --version
 
     p   >>
@@ -233,6 +232,7 @@ page "The YML Toolchain" {
         If «YML_PATH» is set to a colon separated ¬http://en.wikipedia.org/wiki/Path_(computing) path¬
         list, then the `a href="features#including" code "include";` statement searches these paths.
         If «YML_PATH» is not set, the local directory «.» is searched for files.
+        The system location for «.yml2» and «.ysl2» files is always searched afterwards.
         >>
     
     h2 id=recommended > Recommended external tools

yml2/manpage.yml2

@@ -0,0 +1 @@
+decl manpage(xmlns="http://docbook.org/ns/docbook", version="5.0", *title) alias article { title *title; content; };

yml2/yml2c.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+# vim: set fileencoding=utf-8 :
+
+"""\
+YML 2 compiler version 6.2
+Copyleft (c), 2009-2020, Volker Birk  http://fdik.org/yml/
+
+"""
+
+import sys, os, codecs, locale
+import fileinput, unicodedata
+from optparse import OptionParser
+
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.dirname(__file__))
+
+from yml2.pyPEG import parse, u
+from yml2.yml2 import ymlCStyle, comment, oldSyntax
+import yml2.backend as backend
+
+YML_DEFAULT_PATHS = [os.path.dirname(backend.__file__)]
+
+def printInfo(option, opt_str, value, parser):
+    sys.stdout.write(__doc__)
+    sys.exit(0)
+
+def w(msg):
+    if isinstance(msg, BaseException):
+        msg = str(msg) + "\n"
+    if type(msg) is bytes:
+        msg = codecs.encode(msg, sys.stderr.encoding)
+    sys.stderr.write(msg)
+
+def main():
+    dirsepname = {':': 'a colon', ';': 'a semicolon'}.get(os.path.pathsep, "a '%s'" % os.path.pathsep)
+    optParser = OptionParser()
+    optParser.add_option("-C", "--old-syntax", action="store_true", dest="old_syntax",
+            help="syntax of YML 2 version 1.x (compatibility mode)", default=False)
+    optParser.add_option("-D", "--emit-linenumbers", action="store_true", dest="emitlinenumbers",
+            help="emit line numbers into the resulting XML for debugging purposes", default=False)
+    optParser.add_option("-E", "--encoding", dest="encoding", metavar="ENCODING", default=locale.getdefaultlocale()[1],
+            help="encoding of input files (default to locale)")
+    optParser.add_option("-I", "--include", dest="includePathText", metavar="INCLUDE_PATH",
+            help="precede YML_PATH by %s separated INCLUDE_PATH to search for include files" % dirsepname)
+    optParser.add_option("-m", "--omit-empty-parm-tags", action="store_true", dest="omitemptyparm",
+            help="does nothing (only there for compatibility reasons)", default=False)
+    optParser.add_option("-n", "--normalization", dest="normalization", metavar="NORMALIZATION", default="NFC",
+            help="Unicode normalization (none, NFD, NFKD, NFC, NFKC, FCD, default is NFC)")
+    optParser.add_option("-o", "--output", dest="outputFile", metavar="FILE",
+            help="place output in file FILE")
+    optParser.add_option("-p", "--parse-only", action="store_true", dest="parseonly",
+            help="parse only, then output pyAST as text to stdout", default=False)
+    optParser.add_option("-V", "--version", action="callback", callback=printInfo, help="show version info")
+    (options, args) = optParser.parse_args()
+
+    if options.old_syntax:
+        oldSyntax()
+
+    if options.emitlinenumbers:
+        backend.emitlinenumbers = True
+
+    backend.encoding = options.encoding
+
+    try:
+        if options.includePathText:
+            backend.includePath = options.includePathText.split(os.path.pathsep)
+
+        dirs = os.environ.get('YML_PATH', '.').split(os.path.pathsep) + YML_DEFAULT_PATHS
+        backend.includePath.extend(dirs)
+
+        files = fileinput.input(args, mode="rU", openhook=fileinput.hook_encoded(options.encoding))
+
+        ymlC = ymlCStyle()
+        result = parse(ymlC, files, True, comment, packrat=True)
+
+        if options.parseonly:
+            print(result)
+        else:
+            result = backend.finish(result)
+            if options.normalization != "none":
+                result = unicodedata.normalize(options.normalization, result)
+
+            if options.outputFile and options.outputFile != "-":
+                outfile = open(options.outputFile, "wb")
+                outfile.write(codecs.encode(result, options.encoding))
+                outfile.close()
+            else:
+                sys.stdout.buffer.write(codecs.encode(result, options.encoding))
+                print()
+
+    except KeyboardInterrupt:
+        w("\n")
+        sys.exit(1)
+    except KeyError as msg:
+        w("not found: " + u(msg) + "\n")
+        sys.exit(4)
+    except LookupError as msg:
+        w("not found: " + u(msg) + "\n")
+        sys.exit(4)
+    except Exception as msg:
+        w(msg)
+        sys.exit(5)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

yml2/yml2proc.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+# vim: set fileencoding=utf-8 :
+
+"""\
+YML/YSLT 2 processor version 6.2
+Copyleft (c), 2009-2020 Volker Birk  http://fdik.org/yml/
+
+"""
+
+import sys, os, codecs, locale
+import fileinput, unicodedata
+from optparse import OptionParser
+
+try:
+    from lxml import etree
+except:
+    sys.stderr.write("This program needs lxml, see http://codespeak.net/lxml/\n")
+    sys.exit(1)
+
+if __name__ == "__main__":
+    sys.path.insert(0, os.path.dirname(__file__))
+
+from yml2.yml2 import ymlCStyle, comment, oldSyntax
+from yml2.pyPEG import parse, u
+import yml2.backend as backend
+
+YML_DEFAULT_PATHS = [os.path.dirname(backend.__file__)]
+
+def printInfo(option, opt_str, value, parser):
+    sys.stdout.write(__doc__)
+    sys.exit(0)
+
+class YMLAssert(Exception): pass
+
+def w(msg):
+    if isinstance(msg, BaseException):
+        try:
+            msg = str(msg) + "\n"
+        except:
+            msg = u(msg) + "\n"
+    sys.stderr.write(msg)
+
+def main():
+    dirsepname = {':': 'a colon', ';': 'a semicolon'}.get(os.path.pathsep, "a '%s'" % os.path.pathsep)
+    optParser = OptionParser()
+    optParser.add_option("-C", "--old-syntax", action="store_true", dest="old_syntax",
+            help="syntax of YML 2 version 1.x (compatibility mode)", default=False)
+    optParser.add_option("-D", "--emit-linenumbers", action="store_true", dest="emitlinenumbers",
+            help="emit line numbers into the resulting XML for debugging purposes", default=False)
+    optParser.add_option("--debug", action="store_true", dest="trace",
+            help="switch on tracing to stderr", default=False)
+    optParser.add_option("-d", "--paramdict", dest="params", metavar="PARAMS",
+            help="call X/YSLT script with dictionary PARAMS as parameters")
+    optParser.add_option("-e", "--xpath", dest="xpath", metavar="XPATH",
+            help="execute XPath expression XPATH and print result")
+    optParser.add_option("-E", "--encoding", dest="encoding", metavar="ENCODING", default=locale.getdefaultlocale()[1],
+            help="encoding of input files (default to locale)")
+    optParser.add_option("-I", "--include", dest="includePathText", metavar="INCLUDE_PATH",
+            help="precede YML_PATH by %s separated INCLUDE_PATH to search for include files" % dirsepname)
+    optParser.add_option("-m", "--omit-empty-parm-tags", action="store_true", dest="omitemptyparm",
+            help="does nothing (only there for compatibility reasons)", default=False)
+    optParser.add_option("-M", "--empty-input-document", action="store_true", dest="emptyinput",
+            help="use an empty input document", default=False)
+    optParser.add_option("-n", "--normalization", dest="normalization", metavar="NORMALIZATION", default="NFC",
+            help="Unicode normalization (none, NFD, NFKD, NFC, NFKC, FCD, default is NFC)")
+    optParser.add_option("-o", "--output", dest="outputFile", metavar="FILE",
+            help="place output in file FILE")
+    optParser.add_option("-p", "--parse-only", action="store_true", dest="parseonly",
+            help="parse only, then output pyAST as text to stdout", default=False)
+    optParser.add_option("-P", "--pretty", action="store_true", default=False,
+            help="pretty print output adding whitespace")
+    optParser.add_option("-s", "--stringparamdict", dest="stringparams", metavar="STRINGPARAMS",
+            help="call X/YSLT script with dictionary STRINGPARAMS as string parameters")
+    optParser.add_option("-x", "--xml", action="store_true", default=False,
+            help="input document is XML already")
+    optParser.add_option("-X", "--xslt", dest="xslt", metavar="XSLTSCRIPT",
+            help="execute XSLT script XSLTSCRIPT")
+    optParser.add_option("-y", "--yslt", dest="yslt", metavar="YSLTSCRIPT",
+            help="execute YSLT script YSLTSCRIPT")
+    optParser.add_option("-Y", "--xml2yml", action="store_true", default=False,
+            help="convert XML to normalized YML code")
+    optParser.add_option("-V", "--version", action="callback", callback=printInfo, help="show version info and exit")
+    (options, args) = optParser.parse_args()
+
+    if options.old_syntax:
+        oldSyntax()
+
+    if options.trace:
+        backend.enable_tracing = True
+
+    if options.emitlinenumbers:
+        backend.emitlinenumbers = True
+
+    if options.includePathText:
+        backend.includePath = options.includePathText.split(os.path.pathsep)
+
+    backend.encoding = options.encoding
+
+    dirs = os.environ.get('YML_PATH', '.').split(os.path.pathsep) + YML_DEFAULT_PATHS
+    backend.includePath.extend(dirs)
+
+    if options.xml2yml:
+        for directory in backend.includePath:
+            name = os.path.join(directory, "xml2yml.ysl2")
+            if os.path.isfile(name):
+                options.yslt = name
+                options.xml = True
+                break
+        else:
+            sys.stderr.write("Error: Stylesheet xml2yml.ysl2 required for --xml2yml not found\n")
+            sys.stderr.write("Please check your YML_PATH\n")
+            sys.exit(1)
+
+    if  (options.xslt and options.yslt) or (options.xslt and options.xpath) or (options.yslt and options.xpath):
+        sys.stderr.write("Cannot combine --xpath, --xslt and --yslt params\n")
+        sys.exit(1)
+
+    try:
+        ymlC = ymlCStyle()
+
+        rtext = ""
+
+        if not options.emptyinput:
+            files = fileinput.input(args, mode="rU", openhook=fileinput.hook_encoded(options.encoding))
+
+            if options.xml:
+                rtext = ""
+                for line in files:
+                    rtext += line
+            else:
+                result = parse(ymlC, files, True, comment)
+                if options.parseonly:
+                    print(result)
+                    sys.exit(0)
+                else:
+                    rtext = backend.finish(result)
+
+        if not rtext:
+            rtext = "<empty/>"
+
+        def ymldebug(context, text):
+            if options.trace:
+                sys.stderr.write("Debug: " + codecs.encode(u(text), options.encoding) + "\n")
+            return ""
+
+        def ymlassert(context, value, msg):
+            if options.trace:
+                if not value:
+                    raise YMLAssert(msg)
+            return ""
+
+        ymlns = etree.FunctionNamespace("http://fdik.org/yml")
+        ymlns.prefix = "yml"
+        ymlns['debug'] = ymldebug
+        ymlns['assert'] = ymlassert
+
+        if options.xpath:
+            tree = etree.fromstring(rtext)
+            ltree = tree.xpath(codecs.decode(options.xpath, options.encoding))
+            rtext = ""
+            try:
+                for rtree in ltree:
+                    rtext += etree.tostring(rtree, pretty_print=options.pretty, encoding=unicode)
+            except:
+                rtext = ltree
+
+        elif options.yslt or options.xslt:
+            params = {}
+
+            if options.yslt:
+                backend.clearAll()
+                yscript = fileinput.input(options.yslt, mode="rU", openhook=fileinput.hook_encoded(options.encoding))
+                yresult = parse(ymlC, yscript, True, comment)
+                ytext = backend.finish(yresult)
+            else:
+                yscript = fileinput.input(options.xslt, mode="rU")
+                ytext = ""
+                for line in yscript:
+                    ytext += line
+
+            doc = etree.fromstring(rtext)
+
+            xsltree = etree.XML(ytext, base_url=os.path.abspath(yscript.filename()))
+            transform = etree.XSLT(xsltree)
+            
+            if options.params:
+                params = eval(options.params)
+                for key, value in params.iteritems():
+                    if type(value) is not str:
+                        params[key] = u(value)
+            if options.stringparams:
+                for key, value in eval(options.stringparams).iteritems():
+                    params[key] = "'" + u(value) + "'"
+
+            rresult = transform(doc, **params)
+            # lxml is somewhat buggy
+            try:
+                rtext = u(rresult)
+            except:
+                rtext = etree.tostring(rresult, encoding=unicode)
+                if not rtext:
+                    rtext = codecs.decode(str(rresult), "utf-8")
+
+        if options.normalization != "none":
+            rtext = unicodedata.normalize(options.normalization, rtext)
+
+        if options.pretty:
+            plaintext = etree.tostring(etree.fromstring(rtext), pretty_print=True, xml_declaration=True, encoding=options.encoding)
+        else:
+            if isinstance(rtext, str):
+                plaintext = codecs.encode(rtext, options.encoding)
+            else:
+                plaintext = rtext
+
+        try:
+            if plaintext[-1] == "\n":
+                plaintext = plaintext[:-1]
+        except: pass
+
+        if options.outputFile and options.outputFile != "-":
+            outfile = open(options.outputFile, "wb")
+            outfile.write(plaintext)
+            outfile.close()
+        else:
+            sys.stdout.buffer.write(plaintext)
+            if not options.pretty:
+                print()
+
+    except KeyboardInterrupt:
+        w("\n")
+        sys.exit(1)
+    except YMLAssert as msg:
+        w("YML Assertion failed: " + u(msg) + "\n")
+        sys.exit(2)
+    except KeyError as msg:
+        w("not found: " + u(msg) + "\n")
+        sys.exit(4)
+    except LookupError as msg:
+        w("not found: " + u(msg) + "\n")
+        sys.exit(4)
+    except etree.XMLSyntaxError as e:
+        log = e.error_log.filter_from_level(etree.ErrorLevels.FATAL)
+        for entry in log:
+            w("XML error: " + u(entry.message) + "\n")
+        sys.exit(5)
+    except Exception as msg:
+        w(msg)
+        sys.exit(5)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

yml2c

@@ -1,96 +1,10 @@
-#!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
-
-"""\
-YML 2 compiler version 5.9
-Copyleft (c), 2009-2018, Volker Birk  http://fdik.org/yml/
-
-"""
-
-import sys, os, codecs, locale
-import fileinput, unicodedata
-from optparse import OptionParser
-
-from pyPEG import parse, u
-from yml2 import ymlCStyle, comment, oldSyntax
-import backend
-
-def printInfo(option, opt_str, value, parser):
-    sys.stdout.write(__doc__)
-
-def w(msg):
-    if isinstance(msg, BaseException):
-        try:
-            msg = str(msg) + "\n"
-        except:
-            msg = u(msg) + u"\n"
-    if type(msg) is unicode:
-        msg = codecs.encode(msg, sys.stderr.encoding)
-    sys.stderr.write(msg)
-
-optParser = OptionParser()
-optParser.add_option("-C", "--old-syntax", action="store_true", dest="old_syntax",
-        help="syntax of YML 2 version 1.x (compatibility mode)", default=False)
-optParser.add_option("-D", "--emit-linenumbers", action="store_true", dest="emitlinenumbers",
-        help="emit line numbers into the resulting XML for debugging purposes", default=False)
-optParser.add_option("-E", "--encoding", dest="encoding", metavar="ENCODING", default=locale.getdefaultlocale()[1],
-        help="encoding of input files (default to locale)")
-optParser.add_option("-I", "--include", dest="includePathText", metavar="INCLUDE_PATH",
-        help="precede YML_PATH by a colon separated INCLUDE_PATH to search for include files")
-optParser.add_option("-m", "--omit-empty-parm-tags", action="store_true", dest="omitemptyparm",
-        help="does nothing (only there for compatibility reasons)", default=False)
-optParser.add_option("-n", "--normalization", dest="normalization", metavar="NORMALIZATION", default="NFC",
-        help="Unicode normalization (none, NFD, NFKD, NFC, NFKC, FCD, default is NFC)")
-optParser.add_option("-o", "--output", dest="outputFile", metavar="FILE",
-        help="place output in file FILE")
-optParser.add_option("-p", "--parse-only", action="store_true", dest="parseonly",
-        help="parse only, then output pyAST as text to stdout", default=False)
-optParser.add_option("-V", "--version", action="callback", callback=printInfo, help="show version info")
-(options, args) = optParser.parse_args()
-
-if options.old_syntax:
-    oldSyntax()
-
-if options.emitlinenumbers:
-    backend.emitlinenumbers = True
-
-backend.encoding = options.encoding
-
+#!/usr/bin/env python3
+import sys, os
+sys.path.insert(0, os.path.dirname(__file__))
 try:
-    if options.includePathText:
-        backend.includePath = options.includePathText.split(':')
-
-    dirs = os.environ.get('YML_PATH', '.').split(':')
-    backend.includePath.extend(dirs)
-
-    files = fileinput.input(args, mode="rU", openhook=fileinput.hook_encoded(options.encoding))
-
-    ymlC = ymlCStyle()
-    result = parse(ymlC, files, True, comment, packrat=True)
-
-    if options.parseonly:
-        print(result)
-    else:
-        result = backend.finish(result)
-        if options.normalization != "none":
-            result = unicodedata.normalize(options.normalization, result)
-
-        if options.outputFile and options.outputFile != "-":
-            outfile = open(options.outputFile, "w")
-            outfile.write(codecs.encode(result, options.encoding))
-            outfile.close()
-        else:
-            print(codecs.encode(result, options.encoding))
-
-except KeyboardInterrupt:
-    w("\n")
-    sys.exit(1)
-except KeyError as msg:
-    w(u"not found: " + u(msg) + u"\n")
-    sys.exit(4)
-except LookupError as msg:
-    w(u"not found: " + u(msg) + u"\n")
-    sys.exit(4)
-except Exception as msg:
-    w(msg)
-    sys.exit(5)
+    from yml2 import yml2c
+except ImportError as e:
+    import pkg_resources
+    pkg_resources.require('YML2')
+    from yml2 import yml2c
+sys.exit(yml2c.main())

yml2proc

@@ -1,242 +1,10 @@
-#!/usr/bin/env python
-# vim: set fileencoding=utf-8 :
-
-"""\
-YML/YSLT 2 processor version 5.9
-Copyleft (c), 2009-2018 Volker Birk  http://fdik.org/yml/
-
-"""
-
-import sys, os, codecs, locale
-import fileinput, unicodedata
-from optparse import OptionParser
-
+#!/usr/bin/env python3
+import sys, os
+sys.path.insert(0, os.path.dirname(__file__))
 try:
-    from lxml import etree
-except:
-    sys.stderr.write("This program needs lxml, see http://codespeak.net/lxml/\n")
-    sys.exit(1)
-
-from yml2 import ymlCStyle, comment, oldSyntax
-from pyPEG import parse, u
-import backend
-
-def printInfo(option, opt_str, value, parser):
-    sys.stdout.write(__doc__)
-    sys.exit(0)
-
-class YMLAssert(Exception): pass
-
-def w(msg):
-    if isinstance(msg, BaseException):
-        try:
-            msg = str(msg) + "\n"
-        except:
-            msg = u(msg) + u"\n"
-    if type(msg) is unicode:
-        msg = codecs.encode(msg, sys.stderr.encoding)
-    sys.stderr.write(msg)
-
-optParser = OptionParser()
-optParser.add_option("-C", "--old-syntax", action="store_true", dest="old_syntax",
-        help="syntax of YML 2 version 1.x (compatibility mode)", default=False)
-optParser.add_option("-D", "--emit-linenumbers", action="store_true", dest="emitlinenumbers",
-        help="emit line numbers into the resulting XML for debugging purposes", default=False)
-optParser.add_option("--debug", action="store_true", dest="trace",
-        help="switch on tracing to stderr", default=False)
-optParser.add_option("-d", "--paramdict", dest="params", metavar="PARAMS",
-        help="call X/YSLT script with dictionary PARAMS as parameters")
-optParser.add_option("-e", "--xpath", dest="xpath", metavar="XPATH",
-        help="execute XPath expression XPATH and print result")
-optParser.add_option("-E", "--encoding", dest="encoding", metavar="ENCODING", default=locale.getdefaultlocale()[1],
-        help="encoding of input files (default to locale)")
-optParser.add_option("-I", "--include", dest="includePathText", metavar="INCLUDE_PATH",
-        help="precede YML_PATH by a colon separated INCLUDE_PATH to search for include files")
-optParser.add_option("-m", "--omit-empty-parm-tags", action="store_true", dest="omitemptyparm",
-        help="does nothing (only there for compatibility reasons)", default=False)
-optParser.add_option("-M", "--empty-input-document", action="store_true", dest="emptyinput",
-        help="use an empty input document", default=False)
-optParser.add_option("-n", "--normalization", dest="normalization", metavar="NORMALIZATION", default="NFC",
-        help="Unicode normalization (none, NFD, NFKD, NFC, NFKC, FCD, default is NFC)")
-optParser.add_option("-o", "--output", dest="outputFile", metavar="FILE",
-        help="place output in file FILE")
-optParser.add_option("-p", "--parse-only", action="store_true", dest="parseonly",
-        help="parse only, then output pyAST as text to stdout", default=False)
-optParser.add_option("-P", "--pretty", action="store_true", default=False,
-        help="pretty print output adding whitespace")
-optParser.add_option("-s", "--stringparamdict", dest="stringparams", metavar="STRINGPARAMS",
-        help="call X/YSLT script with dictionary STRINGPARAMS as string parameters")
-optParser.add_option("-x", "--xml", action="store_true", default=False,
-        help="input document is XML already")
-optParser.add_option("-X", "--xslt", dest="xslt", metavar="XSLTSCRIPT",
-        help="execute XSLT script XSLTSCRIPT")
-optParser.add_option("-y", "--yslt", dest="yslt", metavar="YSLTSCRIPT",
-        help="execute YSLT script YSLTSCRIPT")
-optParser.add_option("-Y", "--xml2yml", action="store_true", default=False,
-        help="convert XML to normalized YML code")
-optParser.add_option("-V", "--version", action="callback", callback=printInfo, help="show version info and exit")
-(options, args) = optParser.parse_args()
-
-if options.old_syntax:
-    oldSyntax()
-
-if options.trace:
-    backend.enable_tracing = True
-
-if options.emitlinenumbers:
-    backend.emitlinenumbers = True
-
-if options.includePathText:
-    backend.includePath = options.includePathText.split(':')
-
-backend.encoding = options.encoding
-
-dirs = os.environ.get('YML_PATH', '.').split(':')
-backend.includePath.extend(dirs)
-
-if options.xml2yml:
-    for directory in backend.includePath:
-        try:
-            name = directory + "/xml2yml.ysl2"
-            f = open(name, "r")
-            f.close()
-            break
-        except:
-            pass
-
-    options.yslt = name
-    options.xml = True
-
-if  (options.xslt and options.yslt) or (options.xslt and options.xpath) or (options.yslt and options.xpath):
-    sys.stderr.write("Cannot combine --xpath, --xslt and --yslt params\n")
-    sys.exit(1)
-
-try:
-    ymlC = ymlCStyle()
-
-    rtext = u""
-
-    if not options.emptyinput:
-        files = fileinput.input(args, mode="rU", openhook=fileinput.hook_encoded(options.encoding))
-
-        if options.xml:
-            rtext = ""
-            for line in files:
-                rtext += line
-        else:
-            result = parse(ymlC, files, True, comment)
-            if options.parseonly:
-                print(result)
-                sys.exit(0)
-            else:
-                rtext = backend.finish(result)
-
-    if not rtext:
-        rtext = u"<empty/>"
-
-    def ymldebug(context, text):
-        if options.trace:
-            sys.stderr.write("Debug: " + codecs.encode(u(text), options.encoding) + "\n")
-        return ""
-
-    def ymlassert(context, value, msg):
-        if options.trace:
-            if not value:
-                raise YMLAssert(msg)
-        return ""
-
-    ymlns = etree.FunctionNamespace("http://fdik.org/yml")
-    ymlns.prefix = "yml"
-    ymlns['debug'] = ymldebug
-    ymlns['assert'] = ymlassert
-
-    if options.xpath:
-        tree = etree.fromstring(rtext)
-        ltree = tree.xpath(codecs.decode(options.xpath, options.encoding))
-        rtext = u""
-        try:
-            for rtree in ltree:
-                rtext += etree.tostring(rtree, pretty_print=options.pretty, encoding=unicode)
-        except:
-            rtext = ltree
-
-    elif options.yslt or options.xslt:
-        params = {}
-
-        if options.yslt:
-            backend.clearAll()
-            yscript = fileinput.input(options.yslt, mode="rU", openhook=fileinput.hook_encoded(options.encoding))
-            yresult = parse(ymlC, yscript, True, comment)
-            ytext = backend.finish(yresult)
-        else:
-            yscript = fileinput.input(options.xslt, mode="rU")
-            ytext = ""
-            for line in yscript:
-                ytext += line
-
-        doc = etree.fromstring(rtext)
-
-        xsltree = etree.XML(ytext, base_url=os.path.abspath(yscript.filename()))
-        transform = etree.XSLT(xsltree)
-        
-        if options.params:
-            params = eval(options.params)
-            for key, value in params.iteritems():
-                if type(value) != unicode:
-                    params[key] = u(value)
-        if options.stringparams:
-            for key, value in eval(options.stringparams).iteritems():
-                params[key] = u"'" + u(value) + u"'"
-
-        rresult = transform(doc, **params)
-        # lxml is somewhat buggy
-        try:
-            rtext = u(rresult)
-        except:
-            rtext = etree.tostring(rresult, encoding=unicode)
-            if not rtext:
-                rtext = codecs.decode(str(rresult), "utf-8")
-
-    if options.normalization != "none":
-        rtext = unicodedata.normalize(options.normalization, rtext)
-
-    if options.pretty:
-        plaintext = etree.tostring(etree.fromstring(rtext), pretty_print=True, xml_declaration=True, encoding=options.encoding)
-    else:
-        if isinstance(rtext, unicode):
-            plaintext = codecs.encode(rtext, options.encoding)
-        else:
-            plaintext = str(rtext)
-
-    try:
-        if plaintext[-1] == "\n":
-            plaintext = plaintext[:-1]
-    except: pass
-
-    if options.outputFile and options.outputFile != "-":
-        outfile = open(options.outputFile, "w")
-        outfile.write(plaintext)
-        outfile.close()
-    else:
-        print(plaintext)
-
-except KeyboardInterrupt:
-    w("\n")
-    sys.exit(1)
-except YMLAssert as msg:
-    w(u"YML Assertion failed: " + u(msg) + u"\n")
-    sys.exit(2)
-except KeyError as msg:
-    w(u"not found: " + u(msg) + u"\n")
-    sys.exit(4)
-except LookupError as msg:
-    w(u"not found: " + u(msg) + u"\n")
-    sys.exit(4)
-except etree.XMLSyntaxError as e:
-    log = e.error_log.filter_from_level(etree.ErrorLevels.FATAL)
-    for entry in log:
-        w(u"XML error: " + u(entry.message) + u"\n")
-    sys.exit(5)
-except Exception as msg:
-    w(msg)
-    sys.exit(5)
+    from yml2 import yml2proc
+except ModuleNotFoundError:
+    import pkg_resources
+    pkg_resources.require('YML2')
+    from yml2 import yml2proc
+sys.exit(yml2proc.main())