Merge #19772: Do not mutate message when downcoding to 7bit.

Lib/email/generator.py

@@ -12,6 +12,7 @@ import time
 import random
 import warnings
 
+from copy import deepcopy
 from io import StringIO, BytesIO
 from email._policybase import compat32
 from email.header import Header
@@ -173,10 +174,18 @@ class Generator:
         # necessary.
         oldfp = self._fp
         try:
+            self._munge_cte = None
             self._fp = sfp = self._new_buffer()
             self._dispatch(msg)
         finally:
             self._fp = oldfp
+            munge_cte = self._munge_cte
+            del self._munge_cte
+        # If we munged the cte, copy the message again and re-fix the CTE.
+        if munge_cte:
+            msg = deepcopy(msg)
+            msg.replace_header('content-transfer-encoding', munge_cte[0])
+            msg.replace_header('content-type', munge_cte[1])
         # Write the headers.  First we see if the message object wants to
         # handle that itself.  If not, we'll do it generically.
         meth = getattr(msg, '_write_headers', None)
@@ -225,9 +234,14 @@ class Generator:
         if _has_surrogates(msg._payload):
             charset = msg.get_param('charset')
             if charset is not None:
+                # XXX: This copy stuff is an ugly hack to avoid modifying the
+                # existing message.
+                msg = deepcopy(msg)
                 del msg['content-transfer-encoding']
                 msg.set_payload(payload, charset)
                 payload = msg.get_payload()
+                self._munge_cte = (msg['content-transfer-encoding'],
+                                   msg['content-type'])
         if self._mangle_from_:
             payload = fcre.sub('>From ', payload)
         self._write_lines(payload)

Lib/test/test_email/test_contentmanager.py

@@ -536,8 +536,8 @@ class TestRawDataManager(TestEmailBase):
             From: victim@monty.org
             Subject: Help
             Content-Type: text/plain; charset="utf-8"
-            MIME-Version: 1.0
             Content-Transfer-Encoding: base64
+            MIME-Version: 1.0
 
             aidhaSB1biBwcm9ibMOobWUgZGUgcHl0aG9uLiBpbCBlc3Qgc29ydGkgZGUgc29uIHZpdmFyaXVt
             Lgo=

Lib/test/test_email/test_email.py

@@ -3529,7 +3529,7 @@ Here's the message body
         self.assertTrue(msg.get_payload(0).get_payload().endswith('\r\n'))
 
 
-class Test8BitBytesHandling(unittest.TestCase):
+class Test8BitBytesHandling(TestEmailBase):
     # In Python3 all input is string, but that doesn't work if the actual input
     # uses an 8bit transfer encoding.  To hack around that, in email 5.1 we
     # decode byte streams using the surrogateescape error handler, and
@@ -3782,6 +3782,16 @@ class Test8BitBytesHandling(unittest.TestCase):
         email.generator.Generator(out).flatten(msg)
         self.assertEqual(out.getvalue(), self.non_latin_bin_msg_as7bit_wrapped)
 
+    def test_str_generator_should_not_mutate_msg_when_handling_8bit(self):
+        msg = email.message_from_bytes(self.non_latin_bin_msg)
+        out = BytesIO()
+        BytesGenerator(out).flatten(msg)
+        orig_value = out.getvalue()
+        Generator(StringIO()).flatten(msg) # Should not mutate msg!
+        out = BytesIO()
+        BytesGenerator(out).flatten(msg)
+        self.assertEqual(out.getvalue(), orig_value)
+
     def test_bytes_generator_with_unix_from(self):
         # The unixfrom contains a current date, so we can't check it
         # literally.  Just make sure the first word is 'From' and the

Misc/NEWS

@@ -27,6 +27,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #19772: email.generator no longer mutates the message object when
+  doing a down-transform from 8bit to 7bit CTEs.
+
 - Issue #20536: the statistics module now correctly handle Decimal instances
   with positive exponents