#19003: Only replace \r and/or \n line endings in email.generator.

This is a further restoration of backward compatibility, as well as
being correct per the RFCs.

Lib/email/generator.py

@@ -18,6 +18,7 @@ from email.utils import _has_surrogates
 UNDERSCORE = '_'
 NL = '\n'  # XXX: no longer used by the code below.
 
+NLCRE = re.compile(r'\r\n|\r|\n')
 fcre = re.compile(r'^From ', re.MULTILINE)
 
 
@@ -149,14 +150,17 @@ class Generator:
         # We have to transform the line endings.
         if not lines:
             return
-        lines = lines.splitlines(True)
+        lines = NLCRE.split(lines)
         for line in lines[:-1]:
-            self.write(line.rstrip('\r\n'))
-            self.write(self._NL)
-        laststripped = lines[-1].rstrip('\r\n')
-        self.write(laststripped)
-        if len(lines[-1]) != len(laststripped):
+            self.write(line)
             self.write(self._NL)
+        if lines[-1]:
+            self.write(lines[-1])
+        # XXX logic tells me this else should be needed, but the tests fail
+        # with it and pass without it.  (NLCRE.split ends with a blank element
+        # if and only if there was a trailing newline.)
+        #else:
+        #    self.write(self._NL)
 
     def _write(self, msg):
         # We can't write the headers yet because of the following scenario:

Lib/test/test_email/test_email.py

@@ -1598,6 +1598,18 @@ class TestMIMEApplication(unittest.TestCase):
         self.assertEqual(msg.get_payload(), '\uFFFD' * len(bytesdata))
         self.assertEqual(msg2.get_payload(decode=True), bytesdata)
 
+    def test_binary_body_with_unicode_linend_encode_noop(self):
+        # Issue 19003: This is a variation on #16564.
+        bytesdata = b'\x0b\xfa\xfb\xfc\xfd\xfe\xff'
+        msg = MIMEApplication(bytesdata, _encoder=encoders.encode_noop)
+        self.assertEqual(msg.get_payload(decode=True), bytesdata)
+        s = BytesIO()
+        g = BytesGenerator(s)
+        g.flatten(msg)
+        wireform = s.getvalue()
+        msg2 = email.message_from_bytes(wireform)
+        self.assertEqual(msg2.get_payload(decode=True), bytesdata)
+
     def test_binary_body_with_encode_quopri(self):
         # Issue 14360.
         bytesdata = b'\xfa\xfb\xfc\xfd\xfe\xff '

Misc/NEWS

@@ -68,6 +68,10 @@ Core and Builtins
 Library
 -------
 
+
+- Issue #19003:m email.generator now replaces only \r and/or \n line
+  endings, per the RFC, instead of all unicode line endings.
+
 - Issue #28019: itertools.count() no longer rounds non-integer step in range
   between 1.0 and 2.0 to 1.