test: sync_test.py - add --debuglog -d

Complete Sync implementation in python using python threading.

from libpEpAdapter, use:
* Session
* CallbackDispatcher
* Sync Event Queue
    * retrieve_next_sync_event()
    * _inject_sync_event()

*Sync event processing loop*
Write single threaded implementation of the engine function "do_sync_protocol()" using:
* retrieve_next_sync_event() (libpEpAdapter)
* do_sync_protocol_step() (Engine)
The event loop has to be terminated on NULL event.

*Sync Thread*
Implement Sync thread in python. The Thread has to do:
* Init: register_sync_callbacks()
* Sync event processing loop ()
* Cleanup: unregister_sync_callbacks()

*Python API*
The python API stays backwards compatible.
* start_sync() is not allowed to return before "register_sync_callbacks()" in the sync thread has returned
* stop_sync() has to use _inject_sync_event from libpEpAdapter
* is_sync_active() reports if the python Sync thread is running

*Testing*
* Update [~fdik]s sync_test.py / sync_handshake.py to work with the new sync impl.
* Add a test for start_sync() / stop_sync() continuously

.editorconfig

@@ -0,0 +1,30 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = space
+insert_final_newline = true
+max_line_length = 300
+tab_width = 4
+
+# C++ Coding style
+# I know these keys are unknown, regard it as documentation
+[*.{cc,hh}]
+indent_brace_style = K&R
+curly_bracket_next_line = false
+spaces_around_operators = true
+spaces_around_brackets = true
+
+[*.py]
+indent_size = 4
+
+[Makefile]
+indent_style = tab
+
+[*.ini]
+indent_size = 2
+
+[*.cfg]
+indent_size = 2

.gitignore

@@ -0,0 +1,82 @@
+syntax: glob
+
+ws/
+pEp.egg-info/
+dist/
+build/
+.pythonhist
+.gnupg
+.lldb
+.so
+.pEp_management.db*
+.python_history
+__pycache__/
+test/Laptop/
+test/Library/
+test/Phone/
+test/TestInbox/
+test/Backup/
+test/lib/
+test/imap_settings.py
+venv/
+build-windows/Debug/
+build-windows/Release/
+build-windows/packages/
+build-windows/.vs/
+build-windows/pEpPythonAdapter.vcxproj.*
+launch.json
+settings.json
+# Default ignored files
+.idea/
+docs/build/
+.eggs/
+.tox/
+=======
+# Build config
+local.conf
+
+# Python artifacts
+/build/
+/dist/
+__pycache__/
+*.py[co]
+/.tox
+.pytest_cache
+/venv/
+/_venv/
+*.egg-info/
+/*.egg-link
+/*.egg
+/.eggs/
+/pip-wheel-metadata/
+
+# file generated by setuptools_scm
+/src/pEp/__version__.py
+
+# Doc artifacts
+/docs/build/
+
+# Test artifacts
+/test/Laptop
+/test/Library
+/test/Phone
+/test/TestInbox
+/test/Backup
+/test/lib
+/test/imap_settings.py
+
+# Editor artifacts and config
+.idea/
+*~
+.\#*
+\#*#
+*.swp
+.idea
+.spyproject
+*.code-workspace
+tags
+ws
+
+# platform artifacts
+.DS_store
+/src/pEp/_pEp.cpython-38-darwin.so

.hgignore

@@ -1,25 +0,0 @@
-syntax: glob
-
-*.swp
-ws
-tags
-.DS_Store
-pEp.egg-info
-dist
-build
-.pythonhist
-.gnupg
-.lldb
-.pEp_management.db*
-.python_history
-__pycache__
-test/Laptop
-test/Library
-test/Phone
-test/TestInbox
-test/Backup
-test/lib
-test/imap_settings.py
-venv
-launch.json
-settings.json

AUTHORS.txt

@@ -0,0 +1,9 @@
+# This is the list of pEpPythonAdapter's significant contributors.
+#
+# This does not necessarily list everyone who has contributed code
+# To see the full list of contributors, see the revision history in
+# source control.
+Volker Birk <vb@pep-project.org> (original author and maintainer)
+Heck <heck@pep.foundation> (current maintainer and author)
+Hartmut Goebel <h.goebel@crazy-compilers.com> (contributions)
+juga <juga@pep.foundation> (contributions)

DEPENDENCIES

@@ -0,0 +1,6 @@
+# 1st Party Dependencies
+## Prefer git tags instead of SHA hashes when possible.
+
+libpEpAdapter=Release_2.1.17
+pEpEngine=Release_2.1.23
+sequoia=365d00a08bec6a5a48d48a7c7893d78c27092b59

Makefile

@@ -0,0 +1,88 @@
+include Makefile.conf
+
+.PHONY: all compile compile-inplace dist dist-egg dist-whl install install-user venv envtest install-test test develop docs  clean clean-all clean-docs
+all: install
+
+# Build
+# =====
+compile:
+	python3 setup.py build_ext $(DEBUG_OPT) $(PREFIX_OPT)
+
+compile-inplace:
+	python3 setup.py build_ext $(DEBUG_OPT) $(PREFIX_OPT) --inplace
+
+# Packaging
+# =========
+# create wheel and egg package in dist/
+dist: dist-whl dist-egg
+
+# create wheel package in dist/
+dist-whl: compile
+	python3 setup.py bdist_wheel
+
+# create egg package in dist/
+dist-egg: compile
+	python3 setup.py bdist_egg
+
+
+# Installation
+# ============
+# installs the package system wide
+install: compile
+	pip3 install .
+
+# installs the package into your user home
+install-user: compile
+	pip3 install . --user
+
+
+# Envrionment
+# ===========
+# Creates and activates a new venv that has the LD_LIBRARY_PATH/DYLD_LIBRARY_PATH
+# already set for the prefix specified in local.conf
+# Only activates venv if already existing
+venv:
+	python3 -m venv $(VENV_DIR)
+	LD_LIBRARY_PATH=$(PREFIX)/lib \
+	DYLD_LIBRARY_PATH=$(PREFIX)/lib \
+	bash --rcfile $(VENV_DIR)/bin/activate
+
+# Tests if the current environment is able to load the pEp module
+envtest:
+	python3 -c 'import pEp'
+
+# Test
+# ====
+# Use these targets only in venv created with 'make venv'
+install-test: compile
+	pip3 install .[test]
+
+test:
+	pytest
+
+
+# Development
+develop: compile
+	pip install -e .
+
+
+# Documentation
+# =============
+docs: compile-inplace
+	make html -C docs/
+
+
+# Housekeeping
+# ============
+clean-all: clean
+	rm -rf $(VENV_DIR)
+
+clean: clean-docs
+	rm -rf $(BUILD_DIR)
+	rm -rf $(DIST_DIR)
+	rm -rf $(PYTHON_ARTIFACTS)
+	rm -rf $(VERSION_FILE)
+	rm -rf $(BUILD_INPLACE)
+
+clean-docs:
+	make clean -C docs/

Makefile.conf

@@ -0,0 +1,26 @@
+HERE:=$(dir $(lastword $(MAKEFILE_LIST)))
+
+# Constants
+BUILD_DIR = ./build
+DIST_DIR = ./dist
+VERSION_FILE = ./src/pEp/__version__.py
+BUILD_INPLACE = ./src/pEp/_pEp.cpython-38-darwin.so
+PYTHON_ARTIFACTS += ./.eggs
+PYTHON_ARTIFACTS += ./src/pEp.egg-info
+PYTHON_ARTIFACTS += ./.pytest_cache
+VENV_DIR = ./venv
+
+# Build config Defaults
+DEBUG=1
+PREFIX=$(HOME)
+
+######### Overrides #########
+-include $(HERE)local.conf
+
+ifeq ($(DEBUG),1)
+	DEBUG_OPT+=--debug
+endif
+
+ifneq ($(PREFIX),)
+    PREFIX_OPT += --prefix=$(PREFIX)
+endif

README.md

@@ -1,11 +0,0 @@
-There's a setup.py in pEpPythonAdapter/
-
-In Python we've setup tools, so it's:
-
-python3 setup.py build
-python3 setup.py install
-
-That is a Python concept. You can find more about it here:
-
-https://docs.python.org/3/extending/building.html#building-c-and-c-extensions-with-distutils
-

README.rst

@@ -0,0 +1,22 @@
+pEpPythonAdapter
+================
+Python adapter for the `pEpEngine <https://pep.foundation/dev/repos/pEpEngine/>`_
+
+
+Documentation
+-------------
+The build instructions can be found in ``docs/source/install.rst``.
+These build instruction also cover how to generate the complete documentation using sphinx.
+
+Issues
+------
+If you are not pEp internal, please send a mail to: heck@pep.foundation
+
+If you are pEp internal, please open a ticket in our `jira bugtracker <https://pep.foundation/jira/projects/PYADPT/)`_ and for any questions, you are always welcome on #adapter>.
+
+
+License
+-------
+|GNU AFFERO GENERAL PUBLIC LICENSE
+|Version 3, 19 November 2007
+|See LICENSE.txt

build-windows/pEpPythonAdapter.sln

@@ -0,0 +1,31 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.31005.135
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pEpPythonAdapter", "pEpPythonAdapter.vcxproj", "{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Debug|x64.ActiveCfg = Debug|x64
+		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Debug|x64.Build.0 = Debug|x64
+		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Debug|x86.ActiveCfg = Debug|Win32
+		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Debug|x86.Build.0 = Debug|Win32
+		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Release|x64.ActiveCfg = Release|x64
+		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Release|x64.Build.0 = Release|x64
+		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Release|x86.ActiveCfg = Release|Win32
+		{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}.Release|x86.Build.0 = Release|Win32
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {B63BC9BA-EF76-4FB4-9126-29CBFAD9092C}
+	EndGlobalSection
+EndGlobal

build-windows/pEpPythonAdapter.vcxproj

@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <VCProjectVersion>16.0</VCProjectVersion>
+    <ProjectGuid>{F7D4314B-C7BA-4117-9AE7-AC5C1492153D}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Makefile</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="Shared">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <NMakeBuildCommandLine>nmake /F "$(ProjectDir)..\make.mak" debug</NMakeBuildCommandLine>
+    <NMakeOutput>
+    </NMakeOutput>
+    <NMakePreprocessorDefinitions>WIN32;_DEBUG;_DLL;$(NMakePreprocessorDefinitions)</NMakePreprocessorDefinitions>
+    <NMakeCleanCommandLine>nmake /F "$(ProjectDir)..\make.mak" clean</NMakeCleanCommandLine>
+    <SourcePath>$(VC_SourcePath);$(ProjectDir)..</SourcePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <NMakeBuildCommandLine>make.cmd</NMakeBuildCommandLine>
+    <NMakeOutput>pEpPythonAdapter.exe</NMakeOutput>
+    <NMakePreprocessorDefinitions>_DEBUG;$(NMakePreprocessorDefinitions)</NMakePreprocessorDefinitions>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <NMakeBuildCommandLine>nmake /F "$(ProjectDir)..\make.mak" release</NMakeBuildCommandLine>
+    <NMakeOutput>
+    </NMakeOutput>
+    <NMakePreprocessorDefinitions>WIN32;NDEBUG;_DLL;$(NMakePreprocessorDefinitions)</NMakePreprocessorDefinitions>
+    <NMakeCleanCommandLine>nmake /F "$(ProjectDir)..\make.mak" clean</NMakeCleanCommandLine>
+    <SourcePath>$(VC_SourcePath);$(ProjectDir)..</SourcePath>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <NMakeBuildCommandLine>make.cmd</NMakeBuildCommandLine>
+    <NMakeOutput>pEpPythonAdapter.exe</NMakeOutput>
+    <NMakePreprocessorDefinitions>NDEBUG;$(NMakePreprocessorDefinitions)</NMakePreprocessorDefinitions>
+  </PropertyGroup>
+  <ItemDefinitionGroup>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <None Include="..\make.mak" />
+    <None Include="..\setup.py" />
+    <None Include="packages.config" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+    <Import Project="packages\boost.1.72.0.0\build\boost.targets" Condition="Exists('packages\boost.1.72.0.0\build\boost.targets')" />
+    <Import Project="packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets" Condition="Exists('packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets')" />
+    <Import Project="packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets" Condition="Exists('packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets')" />
+  </ImportGroup>
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('packages\boost.1.72.0.0\build\boost.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\boost.1.72.0.0\build\boost.targets'))" />
+    <Error Condition="!Exists('packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\boost_locale-vc142.1.72.0.0\build\boost_locale-vc142.targets'))" />
+    <Error Condition="!Exists('packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\boost_python38-vc142.1.72.0.0\build\boost_python38-vc142.targets'))" />
+  </Target>
+</Project>

build-windows/packages.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="boost" version="1.72.0.0" targetFramework="native" />
+  <package id="boost_locale-vc142" version="1.72.0.0" targetFramework="native" />
+  <package id="boost_python38-vc142" version="1.72.0.0" targetFramework="native" />
+</packages>

dependencies.txt

@@ -1,4 +0,0 @@
-pEpPythonAdapter    => Python 3
-                    => g++ | clang | CL.EXE
-                    -> boost::python
-                    -> pEpEngine

docs/Makefile

@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/source/api/pEp.rst

@@ -0,0 +1,12 @@
+pEp package
+===========
+
+Module contents
+---------------
+
+.. automodule:: pEp
+   :members:
+   :imported-members:
+   :undoc-members:
+   :show-inheritance:
+

docs/source/conf.py

@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../../src'))
+# -- Project information -----------------------------------------------------
+
+project = "pEpPythonAdapter"
+copyright = "2020, Volker Birk, heck, Hartmut Goebel, juga"
+author = "Volker Birk, heck, Hartmut Goebel, juga"
+
+import pEp
+# The full version, including alpha/beta/rc tags
+release = pEp.__version__
+# The short X.Y version
+version = ".".join(release.split(".")[:2])
+
+# DEBUG
+print("release:", release)
+print("version:", version)
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.doctest",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.viewcode",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = ".rst"
+
+# The master toctree document.
+master_doc = "index"
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "nature"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "pEpPythonAdapterdoc"
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (
+        master_doc,
+        "pEpPythonAdapter.tex",
+        "pEpPythonAdapter Documentation",
+        "2020, Volker Birk, heck, juga",
+        "manual",
+    )
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (
+        master_doc,
+        "peppythonadapter",
+        "pEpPythonAdapter Documentation",
+        [author],
+        1,
+    )
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        master_doc,
+        "pEpPythonAdapter",
+        "pEpPythonAdapter Documentation",
+        author,
+        "pEpPythonAdapter",
+        "One line description of project.",
+        "Miscellaneous",
+    )
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ["search.html"]
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {"https://docs.python.org/": None}
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True

docs/source/index.rst

@@ -0,0 +1,21 @@
+.. pEpPythonAdapter documentation master file, created by
+   sphinx-quickstart on Wed Oct 21 12:35:11 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to pEpPythonAdapter's documentation!
+============================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   install
+   software_using
+   api/pEp
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/source/install.rst

@@ -0,0 +1,143 @@
+Installation
+============
+
+Requirements
+------------
+In order to build, use or install this extension module, you need to have some
+system dependencies already installed:
+
+* pEp-base (sequoia, libetpan, asn1c, yml2, pEpEngine, libpEpAdapter)
+* boost-python
+
+These `build instructions <https://dev.pep.foundation/Common%20Adapter%20Documentation/Adapter%20Build%20Instructions>`_ will get you all setup.
+
+Additionally, there is a `build script <http://pep-security.lu/gitlab/juga/Internal-Deployment/-/blob/master/build-pep-stack.sh>`_
+that executes these build instructions automatically (Debian and MacOS):
+
+.. Note:: If you dont install pEp-base system wide, but under a prefix, like /home/foo/local/,
+   you will need to have LD_LIBARY_PATH/DYLD_LIBRARY_PATH adjusted for all the following operations.
+
+
+Build
+-----
+The pEpPythonAdapter is a python extension module that contains C/C++ code that needs to be
+compiled first. So, before any use or installation, the module needs to be built.
+
+Build config
+~~~~~~~~~~~~
+Create a local build config by creating a 'local.conf' file. There is a 'local.conf.example' describing
+all the build options. You can use this as a template.
+
+``cp local.conf.example local.conf``
+
+If you have pEp-base installed under a custom prefix (e.g. /home/foo/local) it is important
+that you specify "PREFIX".
+
+Build
+~~~~~
+To build the module just type:
+
+``make``
+
+This will compile the C/C++ parts of the module and create the python packages in the .egg and .wheel format
+in the dist/ dir.
+
+
+Virtualenv
+----------
+We recommend using a venv to work on/with the pEpPythonAdapter.
+There is a convenience make target that will create and activate a venv that already has the LD_LIBRARY_PATH
+or DYLD_LIBRARY_PATH set according to your ``local.conf``.
+If the venv does not exist yet it will be created and activated.
+If the venv already exists it will only be activated.
+
+``make venv``
+
+After that, to install the pEp module into the venv, do:
+
+``make install``
+
+
+Installation
+------------
+You can install the module in the in the following ways.
+
+To install the extension module system wide or into a venv, use:
+
+``make install``
+
+To install the extension module into your home dir, use:
+
+``make install-user``
+
+
+Test
+----
+To run the whole testsuite you need to first create/activate the venv:
+
+``make venv``
+
+then install the test-dependencies:
+
+``make install-test``
+
+And finally run the test-suite:
+
+``make test``
+
+
+Module Development
+------------------
+To develop on the module itself, first of all create and activate a venv:
+
+``make venv``
+
+Then, in the venv install the module in development mode.
+
+``make develop``
+
+While developing there are two levels of changes. Changes to the python part of the module (pEp), and
+changes to the C/C++ part of the module (_pEp). If you change just python code, the changes are effective immediately.
+If you do changes to the C/C++ part you need to issue ``make develop`` again, to recompile the extension and install
+the new binary (.so/.dylib) of the module into the venv.
+
+Documentation
+-------------
+The documentation of the pEpPythonAdapter uses `Sphinx <https://www.sphinx-doc.org/>`_
+Refer to the `Sphinx installation instructions <https://www.sphinx-doc.org/en/master/usage/installation.html>`_ to install it.
+
+To generate the documentation in the HTML format, there is a make target "docs"
+But first, you need to create/activate the venv or set the LD_LIBRARY_PATH manually.
+
+``make venv``
+
+``make docs``
+
+You can see the generated HTML documentation in a browser opening the directory
+`docs/build/html`.
+
+Housekeeping
+------------
+There are the following "clean" targets.
+
+To delete all the generated documentation, run:
+
+``make docs-clean``
+
+To delete all the "derived" files including eggs, wheels, shared libs, build files and caches, run:
+
+``make clean``
+
+To delete all of make clean plus the venv (should equal a complete reset), run:
+``make clean-all``
+
+
+
+Docker
+------
+If you know how to use docker, you can avoid having to install all
+the dependencies using the image
+https://registry.gitlab.com/juga0/pepdocker/peppythonadapter.
+
+.. Note:: This docker image is not officially maintained and it exists only
+   until there is an official Debian one.

docs/source/software_using.rst

@@ -0,0 +1,7 @@
+Software using pEpPythonAdapter
+===============================
+
+- `pEpProxy <https://pep-security.lu/gitlab/marcel/pepproxy>`_
+- `command-line-tool <https://pep-security.lu/gitlab/enterprise-editon/command-line-tool>`_
+- `pEpPythonMixnet <https://gitea.pep.foundation/pEp.foundation/pEpPythonMixnet>`_
+- `pEpSimulator <URL???>`_

examples/simple_xml_msgformat.py

@@ -0,0 +1,131 @@
+#/!usr/bin/env python3
+#
+# Copyleft 2020, p≡p Security
+# Copyleft 2020, Hartmut Goebel <h.goebel@crazy-compilers.com>
+#
+# This file is under GNU General Public License 3.0
+"""
+This examples shows how to transport pEp messages within a
+simple XML based message format which is capable for attachments.
+
+pEp message elements loosely follow the "pEp for XML specification" and
+https://pEp.software/ns/pEp-1.0.xsd.
+
+IMPORTANT: In this example, no error checking is done. Neither is
+           the "pEp for XML" specification enforced.
+
+Structure of the simple XML based message:
+<msg>
+  <from>from-addr</from>
+  <to>to-addr</to>
+  <body>text of the messgage (must not be XML)</bod>
+  <attachments>
+    <attachment>textual data</attachment>
+    …
+  <attachments>
+</msg>
+"""
+
+from lxml import etree
+import email.utils
+import pEp
+
+__all__ = ["serialize_pEp_message", "parse_serialized_message"]
+
+CT2TAG = {
+    'application/pgp-keys': 'keydata',
+    'application/pEp.sync': 'sync',
+    'application/pEp.distribution': 'distribution',
+    'application/pgp-signature': 'signature',
+}
+
+TAG2CT = dict((v,k)for (k,v) in CT2TAG.items())
+
+PEP_NAMESPACE = "https://pEp.software/ns/pEp-1.0.xsd"
+PEP_NS = '{%s}' % PEP_NAMESPACE
+NSMAP = {'pEp': PEP_NAMESPACE}
+
+INCOMING = 0
+OUTGOING = 1
+
+UNENCRYPTED = 0
+
+def serialize_pEp_message(msg):
+    root = etree.Element("msg", nsmap=NSMAP)
+    etree.SubElement(root, "to").text = str(msg.to[0])
+    etree.SubElement(root, "from").text = str(msg.from_)
+
+    if msg.enc_format == UNENCRYPTED:
+        # unencrypted
+        etree.SubElement(root, "body").text = msg.longmsg
+        attachments = etree.SubElement(root, "attachments")
+        # FIXME: Namespace
+        attachments = etree.SubElement(attachments,
+                                       PEP_NS + "attachments",
+                                       version=msg.opt_fields['X-pEp-Version'])
+        # TODO: opt_fields, esp. auto-consume
+        # TODO: Order pEp attachments by type
+        for attach in msg.attachments:
+            # no need to base64-encode, attachement are ascii-armoured
+            # already
+            #attachment = base64_encode(attachment)
+            # FIXME: Namespace
+            a = etree.SubElement(attachments,
+                                 PEP_NS + CT2TAG[attach.mime_type])
+            a.text = attach.decode("ascii")
+    else: # encrypted
+        # forget about longmsg and original body
+        # encrypted message is an attachment and there might be
+        # further attachments, e.g. new keys
+        # build a new message out of these attachments
+        etree.SubElement(root, "body") # emptry body
+        attachments = etree.SubElement(root, "attachments")
+        assert len(msg.attachments) == 2
+        # first attachment is "Version: 1"
+        attach = msg.attachments[1]
+        # no need to base64-encode, attachements are ascii-armoured
+        # already
+        n = etree.SubElement(root, PEP_NS + "message")
+        n.text = attach.decode("ascii")
+    return etree.tostring(root)
+
+
+def parse_serialized_message(transport_message):
+
+    def addr2identity(text):
+        name, addr = email.utils.parseaddr(text)
+        ident = pEp.Identity(addr, name)
+        ident.update()
+        return ident
+
+    # parse the XML text, fetch from and to
+    root = etree.fromstring(transport_message)
+    from_ = addr2identity(root.xpath("./from/text()")[0])
+    msg1 = pEp.Message(INCOMING, from_)
+    msg1.to.append(addr2identity(root.xpath("./to/text()")[0]))
+    enc_msg = root.find("{%s}message" % PEP_NAMESPACE)
+    if enc_msg is not None:
+        # this is an encrypted message, ignore all but the encrypted message
+        msg1.attachments = [
+            # As of Engine r4652 the encrypted message must be the second
+            # attachment
+            pEp.Blob(b"Version: 1", "application/pgp-encrypted"),
+            pEp.Blob(enc_msg.text.encode(), "application/xxpgp-encrypted")]
+    else:
+        # this is an unencrypted message, might contain pEp attachments
+        msg1.longmsg = root.findtext("body")
+        pEp_attachments = None
+        attachments = root.find("attachments")
+        if attachments is not None:
+            pEp_attachments = attachments.find("{%s}attachments" % PEP_NAMESPACE)
+        if pEp_attachments is not None:
+            msg1.opt_fields['X-pEp-Version'] = pEp_attachments.attrib["version"]
+            pEp_attachs = []
+            for tagname in ("keydata", "signature", "sync", "distribution"):
+                for attach in pEp_attachments.iterfind(
+                    "{%s}%s" % (PEP_NAMESPACE, tagname)):
+                    pEp_attachs.append(
+                        pEp.Blob(attach.text.encode(), TAG2CT[tagname]))
+            msg1.attachments = pEp_attachs
+    msg2, keys, rating, flags = msg1.decrypt()
+    return msg2, rating

local.conf.example

@@ -0,0 +1,11 @@
+# This is an Example build config file (local.conf)
+# you might not need this file, but if the defaults dont work for you
+# You can override them here.