merge "MIME-12" into default branch

src/Makefile

@@ -46,7 +46,7 @@ libpEpMIME.a: pEpMIME.o pEpMIME_internal.o rules.o bodyparser.o \
 unittests: unittest_mime.o unittest_nfc.o unittest_timestamp.o \
     unittest_stringcase.o unittest_toutf8.o unittest_address.o \
     unittest_rule.o unittest_subject.o unittest_base64.o \
-    unittest_qp.o print_message.o \
+    unittest_qp.o print_message.o unittest_mime-12.o \
     gtest-all.o gtest_main.o libpEpMIME.a
 	${CXX} ${LDFLAGS} -L${PREFIX}/lib -o $@ $^ -lpEpAdapter -lpEpEngine -lpthread $(LDLIBS)
 

src/bodyparser.cc

@@ -105,7 +105,16 @@ void add_attachment(message* msg, const BodyLines& body, const MimeHeaders& mh)
         LOG << "ATTACHMENT name=“" << filename << "”\n";
     }
     
-    const std::string content_type = mh.type + '/' + mh.subtype;
+    const std::string content_type = mh.mime_type();
+    if( (msg->attachments==nullptr) && (content_type=="message/rfc822") ) // very special requirement. See MIME-12
+    {
+        const sv forwarded = header_value( mh.tparams, "forwarded");
+        if(forwarded.size())
+        {
+            add_opt_field(msg, Pseudo_Header_Forwarded, forwarded);
+        }
+    }
+    
     bloblist_t* bl = bloblist_add(msg->attachments, decoded, decoded_size, content_type.c_str(), (filename.empty()? nullptr : filename.data()) );
     if(msg->attachments==nullptr)
     {

src/pEpEngine_mime.cc

@@ -6,6 +6,7 @@
 #define _EXPORT_PEP_ENGINE_DLL
 #include <pEp/message.h>
 #include "pEpMIME.hh"
+#include "pEpMIME_internal.hh"
 
 // function interface for pEp engine
 // link this to use pEp MIME with C code
@@ -43,6 +44,38 @@ extern "C"
     }
 
 
+    DYNAMIC_API PEP_STATUS _mime_decode_message_internal(
+            const char *mimetext,
+            size_t size,
+            message **msg,
+            bool* raise_msg_attachment
+        )
+    {
+        assert(msg);
+        assert(mimetext);
+
+        if (!(msg && mimetext))
+            return PEP_ILLEGAL_VALUE;
+
+        PEP_STATUS status = PEP_STATUS_OK;
+        *msg = nullptr;
+
+        try{
+            message* m = pEpMIME::parse_message(mimetext, size, raise_msg_attachment);
+            if(!m)
+            {
+                *msg = m;
+            }else{
+                status = PEP_OUT_OF_MEMORY;
+            }
+        }catch(...)
+        {
+            status = PEP_UNKNOWN_ERROR;
+        }
+        return status;
+    }
+
+
     PEP_STATUS _mime_encode_message_internal(
             const message * msg,
             bool omit_fields,

src/pEpMIME.cc

@@ -120,22 +120,35 @@ message* parse_message2(const char* begin, const char* const end)
 }
 
 
-message* parse_message(const char* mime_text, size_t length)
+message* parse_message(const char* mime_text, size_t length, bool* raise_attachment)
 {
     const char* const end_text = mime_text + length;
     message* msg = parse_message2<is_crlf>(mime_text, end_text);
-    if(msg)
-        return msg;
     
-    msg = parse_message2<is_lf>(mime_text, end_text);
-    if(msg)
-        return msg;
+    if(!msg)
+    {
+        msg = parse_message2<is_lf>(mime_text, end_text);
+    }
+    
+    if(!msg)
+    {
+        msg = parse_message2<is_cr>(mime_text, end_text);
+    }
+    
+    if(!msg)
+    {
+        msg = parse_message2<is_cr_or_lf>(mime_text, end_text);
+    }
+    
+    if(msg && raise_attachment)
+    {
+        stringpair_list_t* forwarded = stringpair_list_find(msg->opt_fields, pEpMIME::Pseudo_Header_Forwarded.data());
+        *raise_attachment = (forwarded && forwarded->value && forwarded->value->value==std::string{"no"});
+        
+        // don't emit this pseudo header to clients.
+        msg->opt_fields = stringpair_list_delete_by_key(msg->opt_fields, pEpMIME::Pseudo_Header_Forwarded.data());
+    }
     
-    msg = parse_message2<is_cr>(mime_text, end_text);
-    if(msg)
-        return msg;
-
-    msg = parse_message2<is_cr_or_lf>(mime_text, end_text);
     return msg;
 }
 

src/pEpMIME.hh

@@ -31,11 +31,12 @@ const unsigned MaxMultipartNestingLevel = 100;
 // parameters:
 //     mime_text (in)       : an "Internet Message"
 //     length (in)          : length of the mime_text, because it might contain NUL bytes
+//     raise_attachment(out): if not nullptr, the value is set to true if the attachment needs to be raised (pEp message format 2.x)
 //
 // return value:
 //     a message struct that must be freed via free_message() or NULL on error.
 //
-message* parse_message(const char* mime_text, size_t length);
+message* parse_message(const char* mime_text, size_t length, bool* raise_attachment = nullptr);
 
 
 // Generates an RFC 5322 compliant Internet Message from the given message struct.

src/pEpMIME_internal.cc

@@ -68,6 +68,8 @@ namespace pEpMIME
 
     const sv Pseudo_Header_Format = ":pEp:MIME:longmsg:format";
     const sv Pseudo_Header_Delsp  = ":pEp:MIME:longmsg:delsp";
+    
+    const sv Pseudo_Header_Forwarded = ":pEp:MIME:attachment1:forwarded";
 
 } // end of namespace pEpMIME
 

src/pEpMIME_internal.hh

@@ -15,7 +15,7 @@
 #   include <string_view>
     typedef std::string_view sv;
 
-#else
+#else // in C++11 / C++14 use boost::string_view instead.
 #   include <boost/utility/string_view.hpp>
     typedef boost::string_view sv;
 
@@ -35,7 +35,7 @@
     
     namespace pEpMIME { using ::operator+=; }
     
-#endif
+#endif // C++17 switch
 
 
 #ifdef LOG_TO_STDERR
@@ -97,8 +97,12 @@ namespace pEpMIME
         return (str!=nullptr) && (str[0] != '\0');
     }
 
+    // extracted from Content-Type of 1st "text/plain" MIME leaf:
     extern const sv Pseudo_Header_Format;
     extern const sv Pseudo_Header_Delsp;
+    
+    // extracted from Content-Type of 1st "message/rfc822" leaf:
+    extern const sv Pseudo_Header_Forwarded;
 
 } // end of namespace pEpMIME
 

src/unittest_mime-12.cc

@@ -0,0 +1,140 @@
+// This file is under GNU General Public License 3.0
+// see LICENSE.txt
+
+#include "pEpMIME.hh"
+#include "pEpMIME_internal.hh" // for Pseudo_Header_Forwarded
+#include "headerparser.hh"
+#include "mime_headers.hh"
+#include "print_message.hh"
+#include "base64.hh"
+#include <pEp/mime.h>
+#include <iostream>
+#include <gtest/gtest.h>
+
+using namespace std::string_literals;
+
+namespace
+{
+    static const sv mail1 =
+        "X-Mozilla-Status: 0001\r\n"
+        "X-Mozilla-Status2: 00000000\r\n"
+        "X-Mozilla-Keys: \r\n"
+        "X-Envelope-From: <alice@pep.lol>\r\n"
+        "X-Envelope-To: <janet@pop.lol>\r\n"
+        "X-Delivery-Time: 1580000000\r\n"
+        "X-Uid: 100\r\n"
+        "Return-Path: <alice@pep.lol>\r\n"
+        "Authentication-Results: lol.com; dmarc=none header.from=pep.lol.com;\r\n"
+        " arc=nonepep-com.aq; dkim=nonepep-com.aq; dkim-adsp=none\r\n"
+        " header.from=\"alice@pep.lol\"pep-com.aq; spf=none\r\n"
+        " smtp.mailfrom=\"alice@pep.lol\"\r\n"
+        "X-Rzg-Expurgate: clean/normal\r\n"
+        "X-Rzg-Expurgate-Id: 111:1111111-11111-11111/0/0\r\n"
+        "X-Rzg-Class-Id: xxx\r\n"
+        "Received-Spf: none\tclient-ip=10.10.10.10;\thelo=\"dragonlord.pep.lol\";\tenvelope-from=\"alice@pep.lol\";\treceiver=smtpin.rzone.de;\tidentity=mailfrom;\r\n"
+        "Received: from dragonlord.pep.lol ([10.10.10.10])\tby smtpin.rzone.de (RZmta 46.4.1\r\n"
+        " OK)\twith ESMTPS id sssssssssssssss\t(using TLSv1.3 with cipher\r\n"
+        " TLS_AES_256_GCM_SHA384 (256 bits))\t(Client did not present a\r\n"
+        " certificate)\tfor <janet@pop.lol>;\tThu, 16 Apr 2020 22:22:22\r\n"
+        " +0200 (CEST)from localhost (localhost [127.0.0.1])\tby dragonlord.pep.lol\r\n"
+        " (Postfix) with ESMTP id aaaaaaaaaaaa\tfor <janet@pop.lol>; Thu,\r\n"
+        " 16 Apr 2020 22:22:22 +0200 (CEST)from dragonlord.pep.lol ([127.0.0.1])\tby\r\n"
+        " localhost (dragonlord.pep.lol [127.0.0.1]) (amavisd-new, port 11111)\twith\r\n"
+        " ESMTP id zzzzzzzzzzzz for <janet@pop.lol>;\tThu, 16 Apr 2020\r\n"
+        " 22:22:22 +0200 (CEST)from pEplol.home\r\n"
+        " (xx  [10.1.111.111])\tby\r\n"
+        " dragonlord.pep.lol (Postfix) with ESMTPSA id 777777777777\tfor\r\n"
+        " <janet@pop.lol>; Thu, 16 Apr 2020 22:22:22 +0200 (CEST)\r\n"
+        "Date: Thu, 16 Apr 2020 22:22:22 +0200\r\n"
+        "From: Alice Keylesst <alice@pep.lol>\r\n"
+        "To: \"janet@pop.lol\" <janet@pop.lol>\r\n"
+        "Subject: =?UTF-8?B?cOKJoXA=?=\r\n"
+        "Message-Id: pEp.example.xxx@pep.foundation\r\n"
+        "In-Reply-To: \r\n"
+        "References: \r\n"
+        "X-Pep-Version: 2.1\r\n"
+        "Mime-Version: 1.0\r\n"
+        "Content-Type: multipart/mixed; boundary=\"=pEp-MIME-12-O=\"\r\n"
+        "Content-Transfer-Encoding: 7bit\r\n"
+        "\r\n"
+        "--=pEp-MIME-12-O=\r\n"
+        "MIME-Version: 1.0\r\n"
+        "Content-Type: multipart/mixed; boundary=\"=pEp-MIME-12-M=\"\r\n"
+        "\r\n"
+        "--=pEp-MIME-12-M=\r\n"
+        "Content-Type: text/plain; charset=\"utf-8\"\r\n"
+        "\r\n"
+        "This message was encrypted with p≡p (https://pep.software). If you are seeing this message,\r\n"
+        "your client does not support raising message attachments. Please click on the message attachment to\r\n"
+        "to view it, or better yet, consider using p≡p!\r\n"
+        "\r\n"
+        "--=pEp-MIME-12-M=\r\n"
+        "Content-Type: message/rfc822; forwarded=\"no\"\r\n"
+        "\r\n"
+        "Message-ID: <<xxxx5zthgbewrte@pretty.Easy.privacy>>\r\n"
+        "Date: Thu, 16 Apr 2020 11:11:11 +0000\r\n"
+        "From: Alice Keylesst <alice@pep.lol>\r\n"
+        "To: \"janet@pop.lol\" <janet@pop.lol>\r\n"
+        "Subject: Test message\r\n"
+        "X-pEp-Version: 2.1\r\n"
+        "X-pEp-Wrapped-Message-Info: INNER\r\n"
+        "X-pEp-Sender-FPR: xxxxxxxxxxxxxxxxxx\r\n"
+        "MIME-Version: 1.0\r\n"
+        "Content-Type: multipart/alternative; boundary=\"=pEp-MIME-12-A=\"\r\n"
+        "\r\n"
+        "--=pEp-MIME-12-A=\r\n"
+        "Content-Type: text/plain; charset=\"utf-8\"\r\n"
+        "Content-Transfer-Encoding: quoted-printable\r\n"
+        "\r\n"
+        "Yay=21 A message=21 =20\r\n"
+        "--  Sent from my p=E2=89=A1p for Android.\r\n"
+        "--=pEp-MIME-12-A=\r\n"
+        "Content-Type: text/html; charset=\"utf-8\"\r\n"
+        "Content-Transfer-Encoding: quoted-printable\r\n"
+        "\r\n"
+        "Yay=21 A message=21 <br>\r\n"
+        "--  Sent from my p=E2=89=A1p for Android.\r\n"
+        "--=pEp-MIME-12-A=--\r\n"
+        "\r\n"
+        "--=pEp-MIME-12-M=\r\n"
+        "Content-Type: application/octet-stream\r\n"
+        "Content-Disposition: attachment; filename=\"lolcat.bin\"\r\n"
+        "\r\n"
+        "3mpygMDV7qzDDz6RRy1bz+HrwUGuNlyCqbsHEjgOE1EiyhblocGFI8ykYG+vzSdW\r\n"
+        "F48Emz15CBVKRTfnkN/QWsxEJZAhTHvHPb8McDpQ0C94cNuRGL1P6+gVaRDHq4fW\r\n"
+        "\r\n"
+        "--=pEp-MIME-12-M=--\r\n"
+        "--=pEp-MIME-12-O=--\r\n"
+        "\r\n";
+}
+
+std::ostream& operator<<(std::ostream& o, const stringpair_list_t* spl)
+{
+    if(spl==nullptr)
+    {
+        return o << "SPL(NULL)";
+    }
+    
+    o << "SPL:\n";
+    while(spl)
+    {
+        o << "\t key=\"" << spl->value->key << "\", value=\"" << spl->value->value << "\"\n";
+        spl = spl->next;
+    }
+    
+    return o;
+}
+
+
+TEST( MimeTest12, Good )
+{
+    bool raise = false;
+    message* m = pEpMIME::parse_message(mail1.data(), mail1.size(), &raise);
+    ASSERT_NE( m, nullptr );
+    
+    EXPECT_TRUE(raise);
+    
+    std::cout << "°°°°°°°°°°°°°°°°\n" << m->opt_fields << "\n++++++++++++++\n";
+    
+    free_message(m);
+}