Issue #29409: Implement PEP 529 for io.FileIO (Patch by Eryk Sun)

Lib/test/test_fileio.py

@@ -9,7 +9,8 @@ from array import array
 from weakref import proxy
 from functools import wraps
 
-from test.support import TESTFN, check_warnings, run_unittest, make_bad_fd, cpython_only
+from test.support import (TESTFN, TESTFN_UNICODE, check_warnings, run_unittest,
+                          make_bad_fd, cpython_only)
 from collections import UserList
 
 import _io  # C implementation of io
@@ -432,6 +433,23 @@ class OtherFileTests:
         finally:
             os.unlink(TESTFN)
 
+    @unittest.skipIf(sys.getfilesystemencoding() != 'utf-8',
+                     "test only works for utf-8 filesystems")
+    def testUtf8BytesOpen(self):
+        # Opening a UTF-8 bytes filename
+        try:
+            fn = TESTFN_UNICODE.encode("utf-8")
+        except UnicodeEncodeError:
+            self.skipTest('could not encode %r to utf-8' % TESTFN_UNICODE)
+        f = self.FileIO(fn, "w")
+        try:
+            f.write(b"abc")
+            f.close()
+            with open(TESTFN_UNICODE, "rb") as f:
+                self.assertEqual(f.read(), b"abc")
+        finally:
+            os.unlink(TESTFN_UNICODE)
+
     def testConstructorHandlesNULChars(self):
         fn_with_NUL = 'foo\0bar'
         self.assertRaises(ValueError, self.FileIO, fn_with_NUL, 'w')

Misc/NEWS

@@ -146,6 +146,8 @@ Library
 Windows
 -------
 
+- Issue #29409: Implement PEP 529 for io.FileIO (Patch by Eryk Sun)
+
 - Issue #29392: Prevent crash when passing invalid arguments into msvcrt module.
 
 - Issue #25778: winreg does not truncate string correctly (Patch by Eryk Sun)

Modules/_io/fileio.c

@@ -230,12 +230,13 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
                          int closefd, PyObject *opener)
 /*[clinic end generated code: output=23413f68e6484bbd input=193164e293d6c097]*/
 {
-    const char *name = NULL;
-    PyObject *stringobj = NULL;
-    const char *s;
 #ifdef MS_WINDOWS
     Py_UNICODE *widename = NULL;
+#else
+    const char *name = NULL;
 #endif
+    PyObject *stringobj = NULL;
+    const char *s;
     int ret = 0;
     int rwa = 0, plus = 0;
     int flags = 0;
@@ -277,24 +278,21 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
         PyErr_Clear();
     }
 
+    if (fd < 0) {
 #ifdef MS_WINDOWS
-    if (PyUnicode_Check(nameobj)) {
         Py_ssize_t length;
-        widename = PyUnicode_AsUnicodeAndSize(nameobj, &length);
-        if (widename == NULL)
-            return -1;
-        if (wcslen(widename) != length) {
-            PyErr_SetString(PyExc_ValueError, "embedded null character");
+        if (!PyUnicode_FSDecoder(nameobj, &stringobj)) {
             return -1;
         }
-    } else
-#endif
-    if (fd < 0)
-    {
+        widename = PyUnicode_AsUnicodeAndSize(stringobj, &length);
+        if (widename == NULL)
+            return -1;
+#else
         if (!PyUnicode_FSConverter(nameobj, &stringobj)) {
             return -1;
         }
         name = PyBytes_AS_STRING(stringobj);
+#endif
     }
 
     s = mode;
@@ -386,11 +384,10 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
             do {
                 Py_BEGIN_ALLOW_THREADS
 #ifdef MS_WINDOWS
-                if (widename != NULL)
-                    self->fd = _wopen(widename, flags, 0666);
-                else
+                self->fd = _wopen(widename, flags, 0666);
+#else
+                self->fd = open(name, flags, 0666);
 #endif
-                    self->fd = open(name, flags, 0666);
                 Py_END_ALLOW_THREADS
             } while (self->fd < 0 && errno == EINTR &&
                      !(async_err = PyErr_CheckSignals()));