MIME-15: remove NUL bytes from text bodies, control chars from header values.

src/bodyparser.cc

@@ -82,11 +82,16 @@ char* create_string(const BodyLines& body, const sv& charset, MimeHeaders::Decod
     
     if(charset=="UTF-8" || charset=="UTF8")
     {
+        // Move all NUL bytes to the end where they don't hurt.
+        std::remove(decoded, decoded + decoded_size, '\0' );
         return decoded; // fine. :-)
     }else{
         // Sigh, the hard way. At the moment with a lot of unecessary copying. :-/
-        // Rule1: Make it work. Profile. Make it fast. In this order.
-        const std::string converted = to_utf8((charset.empty() ? "us-ascii" : charset), sv(decoded, decoded_size) ); // 1st copy...
+        // Rule 1: Make it work. Profile. Make it fast. In this order.
+        std::string converted = to_utf8((charset.empty() ? "us-ascii" : charset), sv(decoded, decoded_size) ); // 1st copy...
+        
+        // remove any NUL bytes
+        converted.erase( std::remove(converted.begin(), converted.end(), '\0'), converted.end() );
         pEp_free(decoded);
         return new_string( converted.data(), converted.size() ); // copy again. :'-(
     }

src/headerparser.cc

@@ -40,7 +40,8 @@ std::string robust_to_utf8(sv s)
     }
     
     // NUL bytes confuse C code, especially the Engine.
-    ret.erase( std::remove(ret.begin(), ret.end(), '\0'), ret.end() );
+    // MIME-15: remove all C0 control characters.
+    ret.erase( std::remove_if(ret.begin(), ret.end(), [](char c){ return uint8_t(c) < ' '; } ), ret.end() );
     return ret;
 }