ENGINE-895: binary-labelled ascii attachments with lines longer than 998 characters will now be base64 encoded in spite of this. (But try to use the right mime-type please)

Reviewed-on: pEp.foundation/pEpEngine#6

.gitignore

@@ -1,14 +1,12 @@
-syntax: regexp
-^asn\.1/.*\.(c|h)$
-^asn\.1/libasn1\.a$
-.*xcuserdata/
-^build/
-^local.conf
-^test_home/
-
-syntax: glob
 asn.1/Makefile.am.*
 asn.1/converter-example.mk
+asn.1/libasn1.a
+local.conf
+build/
+test_home/
+asn.1/*.c
+asn.1/*.h
+xcuserdata/
 *.orig
 *.old
 *.d
@@ -33,6 +31,7 @@ build-android/pEpEngine-android-1.zip
 build-android/third-party
 build-android/pEpEngine-android-1
 build-linux/*
+build-windows/libpEpasn1/libpEpasn1.vcxproj.user
 *.dSYM*
 *_test
 msg4.asc
@@ -89,6 +88,30 @@ test/EngineTests
 test/googletest
 test/tmp/*
 
+src/commit_hash.h
+.idea/
+venv/
+venv2/
+cleangit.py
+newgit
+
+# Xcode related ignores
+build/
+DerivedData/
+*.pbxuser
+!default.pbxuser
+*.mode1v3
+!default.mode1v3
+*.mode2v3
+!default.mode2v3
+*.perspectivev3
+!default.perspectivev3
+xcuserdata/
+*.moved-aside
+*.xccheckout
+*.xcscmblueprint
+xcshareddata
+
 # ignore generated test data
 
 test/655_16F07F382FB3CF5DF977005D1069C7CACF9C23C6.asc

.gitlab-ci-files/common-prepare.yml

@@ -0,0 +1,22 @@
+.ensure_docker: &ensure_docker
+  # Check for docker and install if missing
+  - 'which docker || ( sudo apt-get update -y && sudo apt-get install docker.io -y )'
+
+.ensure_rsync: &ensure_rsync
+  # Install rsync and deps if missing
+  - 'which ssh-agent || ( sudo apt-get update -y && sudo apt-get install openssh-client -y )'
+  - 'which rsync || ( sudo apt-get update -y && sudo apt-get install rsync -y )'
+  - 'which make || ( sudo apt-get update -y && sudo apt-get install make -y )'
+
+.standard_job:
+  tags: [kvm]
+  before_script:
+    - *ensure_docker
+    - *ensure_rsync
+
+.make_in_docker:
+  extends: .standard_job
+  script:
+    - docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST}
+    - cd scripts/${CI_DISTRO_TARGET}
+    - make

.gitlab-ci.yml

@@ -0,0 +1,27 @@
+include:
+  - '.gitlab-ci-files/common-prepare.yml'
+
+stages:
+  - build
+
+
+# Debian
+
+debian10:build:
+  extends: .make_in_docker
+  stage: build
+  variables:
+    CI_DISTRO_TARGET: "debian10"
+    DEBIAN_FRONTEND: "noninteractive"
+  rules:
+    - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+debian10:tagged-build:
+  extends: .make_in_docker
+  stage: build
+  variables:
+    CI_DISTRO_TARGET: "debian10"
+    DEBIAN_FRONTEND: "noninteractive"
+    TAGGED_BUILD: "true"
+  rules:
+    - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'

DEPENDENCIES

@@ -0,0 +1,4 @@
+# 1st Party Dependencies
+## Prefer git tags instead of SHA hashes when possible.
+
+sequoia=365d00a08bec6a5a48d48a7c7893d78c27092b59

Makefile

@@ -19,24 +19,37 @@ ifdef BUILD_CONFIG
     $(info ================================================)
 endif
 
-.PHONY: all sync asn1 build install dbinstall uninstall clean tags test package db
+BUILT_IN_MIME=
 
-build: asn1
+ifdef PEP_MIME
+    BUILT_IN_MIME=pepmime
+endif
+    
+.PHONY: all $(BUILT_IN_MIME) sync asn1 build install dbinstall uninstall clean tags test package db
+
+build: $(BUILT_IN_MIME) asn1
 	$(MAKE) -C src
 
 all: build
 # `make all` is not for tests, that's what `make test` is for
 #	$(MAKE) -C test
 
+pepmime: 
+	$(MAKE) -C pEpMIME lib
+
 sync:
 	$(MAKE) -C sync
 
 asn1: sync
 	$(MAKE) -C asn.1
 
+
 install: build
 	$(MAKE) -C src install
 	$(MAKE) -C asn.1 install
+ifeq ($(BUILT_IN_MIME),pepmime)
+	$(MAKE) -C pEpMIME install
+endif
 
 beinstall:
 	$(MAKE) -C src beinstall
@@ -55,6 +68,9 @@ clean:
 	$(MAKE) -C asn.1 clean
 	$(MAKE) -C sync clean
 	$(MAKE) -C build-android clean
+ifdef PEP_MIME
+	$(MAKE) -C pEpMIME clean
+endif
 
 tags:
 	$(MAKE) -C asn.1 tags

Makefile.conf

@@ -15,7 +15,6 @@
 # It is possible to use multiple variants simultaniously.
 # If nothing is changed according to these 3 methods, a default configuration for your platform (specified below) will be used for the build.
 
-
 ######### Header #########
 HERE:=$(dir $(lastword $(MAKEFILE_LIST)))
 
@@ -74,28 +73,29 @@ TARGET_ARCH=
 # The following two variables will be appended to.
 # You can thus not set them to a final, fixed value here.
 ifeq ($(BUILD_FOR),Linux)
-    LDFLAGS=
+    LDFLAGS+=
 else ifeq ($(BUILD_FOR),Darwin)
     # "-bind_at_load" helps find symbol resolution errors faster
-    LDFLAGS=-bind_at_load
+    LDFLAGS+=-bind_at_load
 endif
 
+# Are we wiping this on purpose?
 LDLIBS=
 
-
 ######### C #########
 ifeq ($(BUILD_FOR),Linux)
-    CC=gcc -std=c99 -pthread
+    CC=gcc 
 else ifeq ($(BUILD_FOR),Darwin)
-    # clang issues a warning when "-pthread" is used for linking.
-    # So, include it in CFLAGS, and not in CC
-    CC=clang -std=c99
+    CC=clang 
 endif
 
+CFLAGS+=-std=c99 -pthread
+CFLAGS+=-fPIC -fstrict-aliasing
+
 ifeq ($(BUILD_FOR),Linux)
-    CFLAGS=-fPIC -fstrict-aliasing -fdiagnostics-color=auto
+    CFLAGS+=-fdiagnostics-color=auto
 else ifeq ($(BUILD_FOR),Darwin)
-    CFLAGS=-pthread -fPIC -fstrict-aliasing -fcolor-diagnostics
+    CFLAGS+=-fcolor-diagnostics
 endif
 
 # The flag -DNDEBUG will always be removed from CFLAGS for compiling tests.
@@ -131,24 +131,28 @@ ifeq ($(BUILD_FOR),Linux)
     # otherwise printed during the compilation of every asn1c-generated C file.
     # It's a glibc specific warning, only present in few versions around ~2.19.
     # See https://lwn.net/Articles/590381/ for a discussion.
-    CFLAGS_GENERATED=-D_DEFAULT_SOURCE
+    CPPFLAGS_GENERATED=-D_DEFAULT_SOURCE
 else ifeq ($(BUILD_FOR),Darwin)
-    CFLAGS_GENERATED=
+    CPPFLAGS_GENERATED=
 endif
 
 
 ######### C++ #########
 ifeq ($(BUILD_FOR),Linux)
-    CXX=g++ -std=gnu++11 -pthread
+    CXX=g++ 
+    CXXFLAGS+=-std=gnu++11
 else ifeq ($(BUILD_FOR),Darwin)
-    # clang issues a warning when "-pthread" is used for linking. So, include it in CXXFLAGS, and not in CXX
-    CXX=clang -std=c++11
+    CXX=clang 
+    CXXFLAGS+=-std=c++11
 endif
 
-# The flag -DNDEBUG will always be removed from CXXFLAGS for compiling tests.
+CXXFLAGS+=-pthread
+
+# The flag -DNDEBUG will always be removed from CPPFLAGS for compiling tests.
+#
 # The tests do not work properly, if compiled with -DNDEBUG
 ifeq ($(BUILD_FOR),Linux)
-    CXXFLAGS=-fdiagnostics-color=auto -I../src -I../asn.1 $(ETPAN_INC)
+    CXXFLAGS+=-fdiagnostics-color=auto -I../src -I../asn.1 $(ETPAN_INC)
     ifdef WARN
         CXXFLAGS+=
     else
@@ -157,10 +161,11 @@ ifeq ($(BUILD_FOR),Linux)
     ifdef DEBUG
         CXXFLAGS+= -g -ggdb
     else
-        CXXFLAGS+= -O3 -DNDEBUG
+        CXXFLAGS+= -O3 
+        CPPFLAGS+= -DNDEBUG
     endif
 else ifeq ($(BUILD_FOR),Darwin)
-    CXXFLAGS=-pthread -fcolor-diagnostics -I../src -I../asn.1 $(ETPAN_INC)
+    CXXFLAGS+=-fcolor-diagnostics -I../src -I../asn.1 $(ETPAN_INC)
     ifdef WARN
         CXXFLAGS+=
     else
@@ -169,16 +174,18 @@ else ifeq ($(BUILD_FOR),Darwin)
     ifdef DEBUG
         CXXFLAGS+= -O0 -g
     else
-        CXXFLAGS+= -O3 -DNDEBUG
+        CXXFLAGS+= -O3 
+        CPPFLAGS+= -DNDEBUG
     endif
 endif
 
 
 ######### C and C++ #########
 ifeq ($(BUILD_FOR),Darwin)
-    CPPFLAGS=-D_DARWIN_C_SOURCE
+    CPPFLAGS+=-D_DARWIN_C_SOURCE
+    LDLIBS+=-liconv
 else
-    CPPFLAGS=
+    LDLIBS+=-luuid
 endif
 
 
@@ -200,39 +207,42 @@ ASN1C_INC=
 #ASN1C_INC=-I$(HOME)/include
 
 
-######### libetpan #########
-# libetpan library search flag
-ETPAN_LIB=
-#ETPAN_LIB=-L$(HOME)/lib
+######### sqlite3 #########
+# If empty (or undefined), compile sqlite3 from the sources shipped with the pEp distribution.
+# Otherwise, use an sqlite3 implementation found in the OS's include/library paths.
+SQLITE3_FROM_OS=placeholder
 
-# libetpan include search flag
-ETPAN_INC=
-#ETPAN_INC=-I$(HOME)/include
 
+######### MIME #########
 
 ######### pEp MIME #########
-# set this to skip libetpan and use PEP_MIME instead
-# PEP_MIME=1
+# Set pEpMIME= anything (there are ifdefs on it) in your local.conf if you want
+# to compile built-in pEpMIME (requires a separate libppEpMIME source repo to be
+# checked out elsewhere on the system - define or redefine these in local.conf
+# if you want to use it; otherwise, we default to etpan. (This will eventually
+# become the default, but not yet.
 #
-# pEp MIME library search flag
-PEP_MIME_LIB=
-# pEp MIME include search flag
-PEP_MIME_INC=
+PEP_MIME=
+PEP_MIME_SRC=$(HOME)/src/pEpMIME/src
 
 
-######### sqlite3 #########
-# If empty (or undefined), compile sqlite3 from the sources shipped with the pEp distribution.
-# Otherwise, use an sqlite3 implementation found in the OS's include/library paths.
-SQLITE3_FROM_OS=placeholder
+######### libetpan #########
+# libetpan library search flag
+#
+ETPAN_LIB=
+#ETPAN_LIB=-L$(HOME)/lib
 
+# libetpan include search flag
+ETPAN_INC=
+#ETPAN_INC=-I$(HOME)/include
 
 ######### OpenPGP #########
 # Selects OpenPGP implementation. must be `SEQUOIA`
 OPENPGP=SEQUOIA
 
 # Sequoia-specific variables
-SEQUOIA_CFLAGS=
-SEQUOIA_LDFLAGS=
+SEQUOIA_CFLAGS+=
+SEQUOIA_LDFLAGS+=
 SEQUOIA_LIB=
 SEQUOIA_INC=
 
@@ -258,6 +268,10 @@ EXCLUDE=
 
 
 ######### Footer #########
+
+################################
+# Include local.conf for any overrides and additional flags
+################################
 -include $(HERE)/local.conf
 
 ifdef BUILD_CONFIG
@@ -265,26 +279,43 @@ ifdef BUILD_CONFIG
 endif
 
 ######### Post processing assignments ########
-# These variables are ineffective when set anywhere else but here.
-# KB: I have no idea why we do this - it totally defeats the purpose of
-#     local.conf.
-#     For now: set if has no value.
+
+# If sequoia has been set up and the SEQUOIA flags aren't defined, set them up.
+# Otherwise, add them to the CFLAGS/CXXFLAGS/LDFLAGS and library/include variables
 ifeq ($(OPENPGP),SEQUOIA)
 ifeq ($(SEQUOIA_CFLAGS),)
-    SEQUOIA_CFLAGS=$(shell pkg-config --cflags-only-other sequoia-openpgp)
+    SEQUOIA_CFLAGS+=$(shell pkg-config --cflags-only-other sequoia-openpgp)
 endif
 ifeq ($(SEQUOIA_LDFLAGS),)
-    SEQUOIA_LDFLAGS=$(shell pkg-config --libs-only-l --libs-only-other sequoia-openpgp)
+    SEQUOIA_LDFLAGS+=$(shell pkg-config --libs-only-l sequoia-openpgp)
 endif
 ifeq ($(SEQUOIA_LIB),)
-    SEQUOIA_LIB=$(shell pkg-config --libs-only-L sequoia-openpgp)
+    SEQUOIA_LIB=$(shell pkg-config --libs-only-L --libs-only-other sequoia-openpgp)
 endif
 ifeq ($(SEQUOIA_INC),)
     SEQUOIA_INC=$(shell pkg-config --cflags-only-I sequoia-openpgp)
 endif
     CFLAGS+= $(SEQUOIA_CFLAGS)
+    CXXFLAGS+= $(SEQUOIA_CFLAGS)
     LD_FLAGS+= $(SEQUOIA_LDFLAGS)
 endif
 
 # YML_PATH is needed in the environment of every call to a program of the YML2 distribution
 export YML_PATH=$(YML2_PATH)
+
+# Postprocess for pEpMIME
+ifdef PEP_MIME
+    # Replace c++11 with c++14 for now (limiting the subst to the ++11 means it doesn't matter if gnu or not)
+    CXXFLAGS:=$(subst ++11,++14,$(CXXFLAGS)) -fPIC -fvisibility=hidden
+endif
+ifdef PEP_MIME
+ifndef PEP_MIME_SRC
+    $(error "Compiling with the PEP_MIME option set requires the value of PEP_MIME_SRC to be set to the source directory for libpEpMIME")
+else
+    CPPFLAGS+=-DPEP_BUILTIN_MIME
+    LDLIBS+=-lstdc++
+    ETPAN_LIB=
+    ETPAN_INC=
+endif
+endif
+

asn.1/Makefile

@@ -5,7 +5,7 @@ include ../Makefile.conf
 
 ALL_SOURCE=$(subst $(NO_SOURCE),,$(wildcard *.c))
 ALL_OBJECTS=$(subst .c,.o,$(ALL_SOURCE))
-CPPFLAGS+=$(CFLAGS_GENERATED)
+CPPFLAGS+=$(CPPFLAGS_GENERATED)
 ASN1C_INC+= -I.
 
 .PHONY: all clean install uninstall
@@ -17,7 +17,7 @@ libasn1.a: $(ALL_OBJECTS)
 	$(AR) -rc $@ $(ALL_OBJECTS)
 
 %.o: %.c %.h
-	$(CC) $(CPPFLAGS) $(CFLAGS) $(OPTIMIZE) $(ASN1C_INC) -c $< -o $@
+	$(CC) $(CFLAGS) $(CPPFLAGS) $(OPTIMIZE) $(ASN1C_INC) -c $< -o $@
 
 Sync.c: sync.asn1 keysync.asn1 pEp.asn1
 	$(ASN1C) -gen-PER -fincludes-quoted -fcompound-names -pdu=auto pEp.asn1 keysync.asn1 $<

asn.1/pEp.asn1

@@ -1,7 +1,7 @@
 -- This file is under BSD License 2.0
 
 -- Sync protocol for p≡p
--- Copyright (c) 2016, 2017 p≡p foundation
+-- Copyright (c) 2016, 2010 p≡p foundation
 
 -- Written by Volker Birk
 
@@ -12,7 +12,7 @@ DEFINITIONS AUTOMATIC TAGS EXTENSIBILITY IMPLIED ::=
 
 BEGIN
 
-EXPORTS Identity, IdentityList, TID, Hash, Version;
+EXPORTS Identity, IdentityList, TID, Hash, Version, Rating;
 
 ISO639-1 ::= PrintableString(FROM ("a".."z")) (SIZE(2))
 Hex ::= PrintableString(FROM ("A".."F") | FROM ("0".."9"))
@@ -36,5 +36,31 @@ Version ::= SEQUENCE {
     minor INTEGER (0..255) DEFAULT 2
 }
 
+Rating ::= ENUMERATED { 
+    -- no color
+
+    cannot-decrypt (1),
+    have-no-key (2),
+    unencrypted (3),
+    unreliable (5),
+
+    b0rken (-2),
+
+    -- yellow
+
+    reliable (6),
+
+    -- green
+
+    trusted (7),
+    trusted-and-anonymized (8),