Fixed decoder generation to create decoder that fails when entire blob isn't consumed (or on other failure), indicating an illegal message

Change SQL definitions.  Add code to migrate cleanly from version 18.

asn.1/Makefile

@@ -19,7 +19,7 @@ STORAGE_FILES      = $(addsuffix .asn1, $(STORAGE))
 
 .PHONY: all clean install uninstall
 
-all: Sync.c Distribution.c Storage.c
+all: Sync.c Distribution.c Storage.c ASN1Message.c
 	$(MAKE) libasn1.a
 
 libasn1.a: $(ALL_OBJECTS)
@@ -43,6 +43,11 @@ Storage.c: $(STORAGE_FILES) pEp.asn1
 	rm -f converter-sample.c
 	touch Storage.c
 
+ASN1Message.c: message.asn1 pEp.asn1
+	$(ASN1C) -gen-PER $(ASN1C_OPTIONS) $+
+	rm -f converter-sample.c
+	touch ASN1Message.c
+
 clean:
 	rm -f *.a *.o *.c *.h *.sample \
 		$(SYNC_FILES) $(DISTRIBUTION_FILES) $(STORAGE_FILES)

asn.1/message.asn1

@@ -0,0 +1,94 @@
+-- This file is under BSD License 2.0
+--
+-- Sync protocol for p≡p
+-- Copyright (c) 2021 p≡p foundation
+--
+-- Written by Hartmut Goebel
+
+
+PEP-MESSAGE {
+    iso(1) org(3) dod(6) internet(1) private(4)
+    enterprise(1) pEp(47878) message(1)
+    }
+DEFINITIONS
+    AUTOMATIC TAGS EXTENSIBILITY IMPLIED ::=
+BEGIN
+
+EXPORTS ASN1Message;
+
+IMPORTS
+  PString, PStringList, Identity, IdentityList, PStringPairList, Hash
+  FROM PEP { iso(1) org(3) dod(6) internet(1) private(4)
+             enterprise(1) pEp(47878) basic(0) };
+
+
+MessageDirection ::= ENUMERATED {
+    incoming (0),
+    outgoing (1)
+}
+
+ContentDisposition ::= ENUMERATED {
+    attachment (0),
+    inline (1),
+    other (2)
+}
+
+PBlob ::= SEQUENCE {
+    value       OCTET STRING (SIZE(0..102400000)),  -- up to 100 MB
+    mime-type   PString OPTIONAL,
+    filename    PString OPTIONAL,
+    disposition ContentDisposition
+}
+PBlobList ::= SEQUENCE OF PBlob
+
+
+-- For the purposes of this profile, GeneralizedTime values MUST be
+-- expressed in Coordinated Universal Time (UTC) and MUST include seconds
+-- (i.e., times are YYYYMMDDHHMMSSZ), even where the number of seconds
+-- is zero.  GeneralizedTime values MUST NOT include fractional seconds.
+
+
+ASN1Message ::= SEQUENCE {
+    direction   MessageDirection OPTIONAL, -- used only in "inner" messages
+    id          PString          OPTIONAL, -- UTF-8 string of message ID
+    sent        GeneralizedTime        OPTIONAL,  -- when the message is sent
+    recv        GeneralizedTime        OPTIONAL,  -- when the message is received
+    from        Identity,                   -- whom the message is from
+    -- At least one of to, cc must not be an empty list
+    -- FIXME: Can this be constrained?
+    to          IdentityList    OPTIONAL,  -- whom the message is to
+    cc          IdentityList    OPTIONAL,  -- whom a CC is being sent
+    bcc         IdentityList    OPTIONAL,  -- whom a BCC is being sent
+    recv-by     Identity        OPTIONAL,  -- via which identity the message
+                                           -- is received
+    reply-to    IdentityList    OPTIONAL,  -- where a reply should go to
+    in-reply-to PStringList     OPTIONAL,  -- list of UTF-8 strings with
+                                           -- MessageIDs of refering messages
+    -- internal: refering_msg_ref
+    references  PStringList     OPTIONAL,  -- list of UTF-8 strings with
+                                           -- references
+    -- internal: refered_by _message-ref-list
+    keywords    PStringList     OPTIONAL, -- list of UTF-8 strings with keywords
+    comments    PString         OPTIONAL, -- UTF-8 string with comments
+    opt-fields  PStringPairList OPTIONAL, -- optional fields
+    sender-fpr  Hash            OPTIONAL,  -- fingerprint of sending signer
+
+    -- At least one of shortmsg, longmsg, longmsg-formatted must be present
+    -- FIXME: Can this be constrained?
+    shortmsg    PString          OPTIONAL, -- UTF-8 string of short message
+
+    -- longmsg must only be used if implementation can handle dynamic allocation
+    longmsg     UTF8String       OPTIONAL, -- UTF-8 string of long message
+                                           -- (plain)
+
+    -- longmsg-formatted must only be used if implementation can handle
+    -- dynamic allocation
+    longmsg-formatted UTF8String   OPTIONAL, -- UTF-8 string of long message
+                                             -- (formatted)
+
+    attachments PBlobList        OPTIONAL     -- blobs with attachments
+    -- internal: rawmsg
+}
+
+END
+

asn.1/pEp.asn1

@@ -12,12 +12,13 @@ DEFINITIONS AUTOMATIC TAGS EXTENSIBILITY IMPLIED ::=
 
 BEGIN
 
-EXPORTS Identity, IdentityList, TID, Hash, Version, Rating;
+EXPORTS Identity, IdentityList, TID, Hash, Version, Rating, PString, PStringList, PStringPair, PStringPairList;
 
 ISO639-1 ::= PrintableString(FROM ("a".."z")) (SIZE(2))
-Hex ::= PrintableString(FROM ("A".."F") | FROM ("0".."9"))
+Hex ::= PrintableString(FROM ("A".."F" | "0".."9"))
 Hash ::= Hex(SIZE(16..128)) -- 32bit Key ID to SHA512 in hex
-PString ::= UTF8String  (SIZE(1..1024))
+PString ::= UTF8String  (SIZE(0..1024))
+PStringList ::= SEQUENCE OF PString
 TID ::= OCTET STRING (SIZE(16)) -- UUID version 4 variant 1
 
 Identity ::= SEQUENCE {
@@ -62,5 +63,12 @@ Rating ::= ENUMERATED {
     under-attack (-3)
 }
 
+PStringPair ::= SEQUENCE {
+    key      PString,
+    value    PString
+}
+
+PStringPairList ::= SEQUENCE OF PStringPair
+
 END
 

codegen/Makefile

@@ -44,5 +44,5 @@ svg: $(patsubst %.dot,%.svg,$(wildcard *.dot))
 	$(YML2_PROC) -y gen_dot.ysl2 $< -o $@
 
 %.svg: %.dot
-	dot -Tsvg -o $@ $<
+	dot -Tsvg $< -o $@ > /dev/null
 

codegen/gen_codec.ysl2

@@ -152,8 +152,13 @@ tstylesheet {
 
             *msg = NULL;
             «@name»_t *_msg = NULL;
-            uper_decode_complete(NULL, &asn_DEF_«@name», (void **) &_msg, data, size);
-            if (!_msg)
+            asn_dec_rval_t rval = uper_decode_complete(NULL, &asn_DEF_«@name», (void **) &_msg, data, size);
+
+            // N.B: If you plan on having messages were the full message isn't consumed by decoding here,
+            // then please look into uper_decode_complete; we still may get a message, even if to contains
+            // nothing. RC_FAIL is an obvious case, but we also need to fail if RC_WMORE is the code, especially
+            // if rval.consumed == 0. Volker, please look into this and decide what you want.
+            if (!_msg || rval.code != RC_OK)
                 return PEP_«yml:ucase(@name)»_ILLEGAL_MESSAGE;
 
             *msg = _msg;

codegen/gen_dot.ysl2

@@ -4,7 +4,7 @@
 include yslt.yml2
 
 tstylesheet {
-    template "protocol/fsm[count(state)>0]" document "{@name}.dot", "text"
+    template "/protocol/fsm[count(state)>0]"
     ||
     digraph finite_state_machine {
         rankdir=LR;

doc/build-debian.md

@@ -1,43 +1,56 @@
-<!-- Copyright 2015-2017, pEp foundation, Switzerland
+<!-- Copyright 2015-2021, pEp foundation, Switzerland
 This file is part of the pEp Engine
 This file may be used under the terms of the Creative Commons Attribution-ShareAlike 3.0 Unported (CC BY-SA 3.0) License
 See CC_BY-SA.txt -->
 
-# Build instructions for Debian 9
+# Build instructions for Debian 9 and 10
+
+We assume the user keeps sources under `~/pep-src`.  There is no single
+installation prefix: each package is built, or when supported installed, in
+either its source directory or in a subdirectory of its source directory.
+Rationale: we do not pollute the user system, uninstalling is trivial.
+
+This is Unix: we assume no spaces in user names.
+
+~~~
+mkdir -p ~/pep-src
+~~~
 
 # Installing packaged dependencies
 
 ~~~
 # general
-apt install -y mercurial
-# YML2
-apt install -y python-lxml
+apt install -y git
 # libetpan
 apt install -y git build-essential automake libtool
 # asn1c
 apt install -y git build-essential automake libtool autoconf
+# sequoia
+apt install git rustc cargo clang libclang-dev make pkg-config nettle-dev libssl-dev capnproto libsqlite3-dev
 # engine
 apt install -y uuid-dev libgpgme-dev libsqlite3-dev sqlite3
+# optional: developer documentation
+apt install -y doxygen pandoc
 ~~~
 
 # Installing unpackaged dependencies
 ## YML2
 
 ~~~
-mkdir -p ~/code/yml2
-git clone https://gitea.pep.foundation/fdik/yml2.git ~/code/yml2
+mkdir -p ~/pep-src/yml2
+git clone https://gitea.pep.foundation/fdik/yml2.git ~/pep-src/yml2
 ~~~
 
 ## libetpan
 pEp Engine requires libetpan with a set of patches that have not been upstreamed yet.
 
 ~~~
-mkdir -p ~/code/libetpan
+mkdir -p ~/pep-src/libetpan
 
-git clone https://gitea.pep.foundation/pEp.foundation/libetpan.git ~/code/libetpan
-cd ~/code/libetpan
-mkdir ~/code/libetpan/build
-./autogen.sh --prefix="$HOME/code/libetpan/build"
+git clone https://gitea.pep.foundation/pEp.foundation/libetpan.git ~/pep-src/libetpan
+cd ~/pep-src/libetpan
+mkdir ~/pep-src/libetpan/build
+./autogen.sh --prefix="$HOME/pep-src/libetpan/build"
 make
 make install
 ~~~
@@ -45,45 +58,70 @@ make install
 ## asn1c
 
 ~~~
-mkdir -p ~/code/asn1c
-git clone git://github.com/vlm/asn1c.git ~/code/asn1c
-cd ~/code/asn1c
+mkdir -p ~/pep-src/asn1c
+git clone git://github.com/vlm/asn1c.git ~/pep-src/asn1c
+cd ~/pep-src/asn1c
 git checkout tags/v0.9.28 -b pep-engine
 autoreconf -iv
-mkdir ~/code/asn1c/build
-./configure --prefix="$HOME/code/asn1c/build"
+mkdir ~/pep-src/asn1c/build
+./configure --prefix="$HOME/pep-src/asn1c/build"
 make
 make install
 ~~~
 
+## sequoia
+
+~~~
+git clone https://gitlab.com/sequoia-pgp/sequoia
+cd ~/pep-src/sequoia
+git checkout openpgp/v1.3.0
+# Make an optimised sequoia build.
+cargo build --all --release -j16
+~~~
+
+This alternative for the last line above is faster, but generates compiled libraries
+in `~/pep-src/sequoia/target/debug` instead of `~/pep-src/sequoia/target/release`:
+several definitions below need to be adapted.
+~~~
+# Alternative: make a debugging sequoia build.
+cargo build --all -j16
+~~~
+
 # pEp Engine
 
 ~~~
-mkdir -p ~/code/pep-engine
-hg clone https://pep.foundation/dev/repos/pEpEngine/ ~/code/pep-engine
-cd ~/code/pep-engine
-mkdir ~/code/pep-engine/build
+mkdir -p ~/pep-src/pep-engine
+git clone https://gitea.pep.foundation/pEp.foundation/pEpEngine ~/pep-src/pep-engine
+cd ~/pep-src/pep-engine
+mkdir build
 ~~~
 
-Edit the build configuration to your needs in `Makefile.conf`, or create a `local.conf` that sets any of the make variables documented in `Makefile.conf`. All the default values for the build configuration variables on each platform are documented in `Makefile.conf`.
+Edit the build configuration to your needs in `Makefile.conf`, or create a `local.conf` in your source directory (the same containing `Makefile.conf`) that sets any of the make variables documented in `Makefile.conf`. All the default values for the build configuration variables on each platform are documented in `Makefile.conf`.
 
 If a dependency is not found in your system's default include or library paths, you will have to specify the according paths in a make variable. Typically, this has to be done at least for YML2, libetpan and asn1c.
 
 For a more detailed explanation of the mechanics of these build configuration files, and overriding defaults, see the comments in `Makefile.conf`.
 
-Below is a sample `./local.conf` file, for orientation.
+The following `./local.conf` example should work in the configuration described here.
 
 ~~~
-PREFIX=$(HOME)/code/pep-engine/build
+PREFIX=$(HOME)/pep-src/pep-engine/build
 PER_MACHINE_DIRECTORY=$(PREFIX)/share/pEp
 
-YML2_PATH=$(HOME)/code/yml2
+YML2_PATH=$(HOME)/pep-src/yml2
+
+ETPAN_LIB=-L$(HOME)/pep-src/libetpan/build/lib
+ETPAN_INC=-I$(HOME)/pep-src/libetpan/build/include
+
+ASN1C=$(HOME)/pep-src/asn1c/build/bin/asn1c
+ASN1C_INC=-I$(HOME)/pep-src/asn1c/build/share/asn1c
 
-ETPAN_LIB=-L$(HOME)/code/libetpan/build/lib
-ETPAN_INC=-I$(HOME)/code/libetpan/build/include
+SEQUOIA_INC=-I$(HOME)/pep-src/sequoia/openpgp-ffi/include
+SEQUOIA_LDFLAGS=-L$(HOME)/pep-src/sequoia/target/release
 
-ASN1C=$(HOME)/code/asn1c/build/bin/asn1c
-ASN1C_INC=-I$(HOME)/code/asn1c/build/share/asn1c
+GTEST_SRC_DIR=$(HOME)/pep-src/googletest/googletest
+GTEST_INC_DIR=$(HOME)/pep-src/googletest/googletest/include
+GTEST_PL=$(HOME)/pep-src/gtest-parallel/gtest_parallel.py
 ~~~
 
 The engine is built as follows:

src/engine_sql.c

@@ -713,7 +713,7 @@ static PEP_STATUS _create_core_tables(PEP_SESSION session) {
             "   timestamp integer default (datetime('now')),\n"
             "   primary key (address, user_id)\n"
             ");\n"
-            "create index if not exists identity_userid_addr on identity(address, user_id);\n"
+            "create index if not exists identity_userid on identity (user_id);\n"
             "create table if not exists trust (\n"
             "   user_id text not null\n"
             "       references person (id)\n"
@@ -1548,6 +1548,27 @@ static PEP_STATUS _upgrade_DB_to_ver_18(PEP_SESSION session) {
     return _force_upgrade_own_latest_message_version(session);
 }
 
+static PEP_STATUS _upgrade_DB_to_ver_19(PEP_SESSION session) {
+    int int_result = sqlite3_exec(
+            session->db,
+            /* This index was useless: it was an index on the (multi-column)
+               primary key, always implemented using an index which gets also
+               used in queries. */
+            "drop index if exists identity_userid_addr;\n"
+            "\n"
+            "create index if not exists identity_userid on identity (user_id);\n",
+            NULL,
+            NULL,
+            NULL
+    );
+    assert(int_result == SQLITE_OK);
+
+    if (int_result != SQLITE_OK)
+        return PEP_UNKNOWN_DB_ERROR;
+
+    return PEP_STATUS_OK;
+}
+
 // Honestly, the upgrades should be redone in a transaction IMHO.
 static PEP_STATUS _check_and_execute_upgrades(PEP_SESSION session, int version) {
     PEP_STATUS status = PEP_STATUS_OK;
@@ -1620,6 +1641,10 @@ static PEP_STATUS _check_and_execute_upgrades(PEP_SESSION session, int version)
             if (status != PEP_STATUS_OK)
                 return status;
         case 18:
+            status = _upgrade_DB_to_ver_19(session);
+            if (status != PEP_STATUS_OK)
+                return status;
+        case 19:
             break;
         default:
             return PEP_ILLEGAL_VALUE;

src/engine_sql.h

@@ -3,7 +3,7 @@
 #include "pEp_internal.h"
 
 // increment this when patching DDL
-#define _DDL_USER_VERSION "18"
+#define _DDL_USER_VERSION "19"
 
 PEP_STATUS init_databases(PEP_SESSION session);
 PEP_STATUS pEp_sql_init(PEP_SESSION session);

src/map_asn1.c

@@ -131,8 +131,7 @@ IdentityList_t *IdentityList_from_identity_list(
 {
     bool allocated = !result;
 
-    assert(list);
-    if (!list)
+    if (!(list && list->ident))
         return NULL;
 
     if (allocated) {
@@ -190,3 +189,902 @@ enomem:
     return NULL;
 }
 
+PStringPair_t *PStringPair_from_Struct(
+        const stringpair_t *value,
+        PStringPair_t *result
+    )
+{
+    bool allocated = !result;
+
+    assert(value);
+    if (!value)
+        return NULL;
+
+    if (allocated)
+        result = (PStringPair_t *) calloc(1, sizeof(PStringPair_t));
+    assert(result);
+    if (!result)
+        return NULL;
+    
+    if (value->key) {
+        int r = OCTET_STRING_fromBuf(&result->key, value->key, -1);
+        if (r)
+            goto enomem;
+    }
+
+    if (value->value) {
+        int r = OCTET_STRING_fromBuf(&result->value, value->value, -1);
+        if (r)
+            goto enomem;
+    }
+
+    return result;
+
+enomem:
+    if (allocated)
+        ASN_STRUCT_FREE(asn_DEF_PStringPair, result);
+    return NULL;
+}
+
+stringpair_t *PStringPair_to_Struct(PStringPair_t *value)
+{
+    assert(value);
+    if (!value)
+        return NULL;
+
+    stringpair_t *result = (stringpair_t *) calloc(1, sizeof(stringpair_t));
+    assert(result);
+    if (!result)
+        goto enomem;
+
+    result->key = strndup((char *) value->key.buf,
+            value->key.size);
+    assert(result->key);
+    if (!result->key)
+        goto enomem;
+
+    result->value = strndup((char *) value->value.buf,
+            value->value.size);
+    assert(result->value);
+    if (!result->value)
+        goto enomem;
+
+    return result;
+
+enomem:
+    free_stringpair(result);
+    return NULL;
+}
+
+PStringPairList_t *PStringPairList_from_stringpair_list(
+        const stringpair_list_t *list,
+        PStringPairList_t *result
+    )
+{
+    bool allocated = !result;
+
+    assert(list);
+    if (!(list && list->value))
+        return NULL;
+
+    if (allocated) {
+        result = (PStringPairList_t *) calloc(1, sizeof(PStringPairList_t));
+        assert(result);
+        if (!result)
+            return NULL;
+    }
+    else {
+        asn_sequence_empty(result);
+    }
+
+    for (const stringpair_list_t *l = list; l && l->value; l=l->next) {
+        PStringPair_t *value = PStringPair_from_Struct(l->value, NULL);
+        if (ASN_SEQUENCE_ADD(&result->list, value)) {
+            ASN_STRUCT_FREE(asn_DEF_PStringPair, value);
+            goto enomem;
+        }
+    }
+
+    return result;
+
+enomem:
+    if (allocated)
+        ASN_STRUCT_FREE(asn_DEF_PStringPairList, result);
+    return NULL;
+}
+
+stringpair_list_t *PStringPairList_to_stringpair_list(
+        PStringPairList_t *list,
+        stringpair_list_t *result
+    )
+{
+    bool allocated = !result;
+
+    assert(list);
+    if (!list)
+        return NULL;
+
+    if (allocated)
+        result = new_stringpair_list(NULL);
+    if (!result)
+        return NULL;
+
+    stringpair_list_t *r = result;
+    for (int i=0; i<list->list.count; i++) {
+        stringpair_t *value = PStringPair_to_Struct(list->list.array[i]);
+        r = stringpair_list_add(r, value);
+        if (!r)
+            goto enomem;
+    }
+
+    return result;
+
+enomem:
+    if (allocated)
+        free_stringpair_list(result);
+    return NULL;
+}
+
+PStringList_t *PStringList_from_stringlist(
+        const stringlist_t *list,
+        PStringList_t *result
+    )
+{
+    bool allocated = !result;
+
+    assert(list);
+    if (!(list && list->value))
+        return NULL;
+
+    if (allocated) {
+        result = (PStringList_t *) calloc(1, sizeof(PStringList_t));
+        assert(result);
+        if (!result)
+            return NULL;
+    }
+    else {
+        asn_sequence_empty(result);
+    }
+
+    for (const stringlist_t *l = list; l && l->value; l=l->next) {
+        PString_t *element = (PString_t *) calloc(1, sizeof(PString_t));
+        assert(element);
+        if (!element)
+            goto enomem;
+
+        int r = OCTET_STRING_fromBuf(element, l->value, -1);
+        if (r)
+            goto enomem;
+        if (ASN_SEQUENCE_ADD(&result->list, element)) {
+            ASN_STRUCT_FREE(asn_DEF_PString, element);
+            goto enomem;
+        }
+    }
+
+    return result;
+
+enomem:
+    if (allocated)
+        ASN_STRUCT_FREE(asn_DEF_PStringList, result);
+    return NULL;
+}
+
+stringlist_t *PStringList_to_stringlist(PStringList_t *list)
+{
+    assert(list);
+    if (!list)
+        return NULL;
+
+    stringlist_t *result = (stringlist_t *) calloc(1, sizeof(stringlist_t));
+    assert(result);
+    if (!result)
+        goto enomem;
+
+    stringlist_t *r = result;
+
+    for (int i=0; i<list->list.count; i++) {
+        char *s = strndup((char *) list->list.array[i]->buf,
+                list->list.array[i]->size);
+        assert(s);
+        if (!s)
+            goto enomem;
+        r->value = s;
+        if (i < list->list.count-1) {
+            r->next = (stringlist_t *) calloc(1, sizeof(stringlist_t));
+            assert(r->next);
+            if (!r->next)
+                goto enomem;
+            r = r->next;
+        }
+    }
+
+    return result;
+
+enomem:
+    free_stringlist(result);
+    return NULL;
+}
+
+PBlobList_t *PBlobList_from_bloblist(
+        bloblist_t *list,
+        PBlobList_t *result,
+        bool copy,
+        size_t max_blob_size
+    )
+{