2000-09-04 02:18:13 +08:00
|
|
|
/*
|
|
|
|
+----------------------------------------------------------------------+
|
2014-09-20 00:33:14 +08:00
|
|
|
| PHP Version 7 |
|
2000-09-04 02:18:13 +08:00
|
|
|
+----------------------------------------------------------------------+
|
2018-01-02 12:57:58 +08:00
|
|
|
| Copyright (c) 1997-2018 The PHP Group |
|
2000-09-04 02:18:13 +08:00
|
|
|
+----------------------------------------------------------------------+
|
2006-01-01 20:51:34 +08:00
|
|
|
| This source file is subject to version 3.01 of the PHP license, |
|
2000-09-04 02:18:13 +08:00
|
|
|
| that is bundled with this package in the file LICENSE, and is |
|
2003-06-25 13:05:06 +08:00
|
|
|
| available through the world-wide-web at the following url: |
|
2006-01-01 20:51:34 +08:00
|
|
|
| http://www.php.net/license/3_01.txt |
|
2000-09-04 02:18:13 +08:00
|
|
|
| If you did not receive a copy of the PHP license and are unable to |
|
|
|
|
| obtain it through the world-wide-web, please send a note to |
|
|
|
|
| license@php.net so we can mail you a copy immediately. |
|
|
|
|
+----------------------------------------------------------------------+
|
2018-11-02 00:35:32 +08:00
|
|
|
| Authors: Andi Gutmans <andi@php.net> |
|
2000-09-04 02:18:13 +08:00
|
|
|
| Sascha Schumann <sascha@schumann.cx> |
|
2010-09-01 17:49:53 +08:00
|
|
|
| Pierre Joye <pierre@php.net> |
|
2000-09-04 02:18:13 +08:00
|
|
|
+----------------------------------------------------------------------+
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/types.h>
|
|
|
|
#include <sys/stat.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <limits.h>
|
|
|
|
#include <errno.h>
|
|
|
|
#include <stdlib.h>
|
|
|
|
#include <fcntl.h>
|
2004-10-07 07:56:29 +08:00
|
|
|
#include <time.h>
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2013-10-17 16:40:43 +08:00
|
|
|
#include "zend.h"
|
2013-10-17 15:43:52 +08:00
|
|
|
#include "zend_virtual_cwd.h"
|
2000-09-04 02:47:35 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2000-09-04 02:47:35 +08:00
|
|
|
#include <io.h>
|
2001-04-28 00:41:53 +08:00
|
|
|
#include "tsrm_win32.h"
|
2009-06-17 05:54:12 +08:00
|
|
|
# ifndef IO_REPARSE_TAG_SYMLINK
|
|
|
|
# define IO_REPARSE_TAG_SYMLINK 0xA000000C
|
|
|
|
# endif
|
2010-09-17 19:34:49 +08:00
|
|
|
|
2012-11-01 05:41:03 +08:00
|
|
|
# ifndef IO_REPARSE_TAG_DEDUP
|
|
|
|
# define IO_REPARSE_TAG_DEDUP 0x80000013
|
|
|
|
# endif
|
|
|
|
|
2017-11-08 18:52:42 +08:00
|
|
|
# ifndef IO_REPARSE_TAG_CLOUD
|
|
|
|
# define IO_REPARSE_TAG_CLOUD (0x9000001AL)
|
|
|
|
# endif
|
|
|
|
/* IO_REPARSE_TAG_CLOUD_1 through IO_REPARSE_TAG_CLOUD_F have values of 0x9000101AL
|
|
|
|
to 0x9000F01AL, they can be checked against the mask. */
|
|
|
|
#ifndef IO_REPARSE_TAG_CLOUD_MASK
|
|
|
|
#define IO_REPARSE_TAG_CLOUD_MASK (0x0000F000L)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifndef IO_REPARSE_TAG_ONEDRIVE
|
|
|
|
#define IO_REPARSE_TAG_ONEDRIVE (0x80000021L)
|
|
|
|
#endif
|
|
|
|
|
2020-04-24 23:24:25 +08:00
|
|
|
# ifndef IO_REPARSE_TAG_ACTIVISION_HSM
|
|
|
|
# define IO_REPARSE_TAG_ACTIVISION_HSM (0x00000047L)
|
|
|
|
# endif
|
|
|
|
|
|
|
|
# ifndef IO_REPARSE_TAG_PROJFS
|
|
|
|
# define IO_REPARSE_TAG_PROJFS (0x9000001CL)
|
|
|
|
# endif
|
|
|
|
|
2010-09-17 19:34:49 +08:00
|
|
|
# ifndef VOLUME_NAME_NT
|
|
|
|
# define VOLUME_NAME_NT 0x2
|
|
|
|
# endif
|
|
|
|
|
|
|
|
# ifndef VOLUME_NAME_DOS
|
|
|
|
# define VOLUME_NAME_DOS 0x0
|
|
|
|
# endif
|
2000-09-04 02:18:13 +08:00
|
|
|
#endif
|
|
|
|
|
2002-11-07 02:07:23 +08:00
|
|
|
#ifndef HAVE_REALPATH
|
2002-10-05 19:26:17 +08:00
|
|
|
#define realpath(x,y) strcpy(y,x)
|
|
|
|
#endif
|
|
|
|
|
2000-09-04 02:18:13 +08:00
|
|
|
#define VIRTUAL_CWD_DEBUG 0
|
|
|
|
|
|
|
|
#include "TSRM.h"
|
|
|
|
|
2016-11-12 18:20:01 +08:00
|
|
|
/* Only need mutex for popen() in Windows because it doesn't chdir() on UNIX */
|
|
|
|
#if defined(ZEND_WIN32) && defined(ZTS)
|
2000-09-04 02:18:13 +08:00
|
|
|
MUTEX_T cwd_mutex;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef ZTS
|
2004-10-05 08:42:25 +08:00
|
|
|
ts_rsrc_id cwd_globals_id;
|
2000-09-04 02:18:13 +08:00
|
|
|
#else
|
2004-10-05 08:42:25 +08:00
|
|
|
virtual_cwd_globals cwd_globals;
|
2000-09-04 02:18:13 +08:00
|
|
|
#endif
|
|
|
|
|
2018-03-18 04:17:47 +08:00
|
|
|
static cwd_state main_cwd_state; /* True global */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifndef ZEND_WIN32
|
2000-09-04 02:18:13 +08:00
|
|
|
#include <unistd.h>
|
|
|
|
#else
|
|
|
|
#include <direct.h>
|
2019-08-20 01:44:37 +08:00
|
|
|
#include "zend_globals.h"
|
|
|
|
#include "zend_globals_macros.h"
|
2000-09-04 02:18:13 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#define CWD_STATE_COPY(d, s) \
|
|
|
|
(d)->cwd_length = (s)->cwd_length; \
|
2013-10-17 16:40:43 +08:00
|
|
|
(d)->cwd = (char *) emalloc((s)->cwd_length+1); \
|
2000-09-04 02:18:13 +08:00
|
|
|
memcpy((d)->cwd, (s)->cwd, (s)->cwd_length+1);
|
|
|
|
|
|
|
|
#define CWD_STATE_FREE(s) \
|
2016-10-24 20:36:17 +08:00
|
|
|
efree((s)->cwd); \
|
|
|
|
(s)->cwd_length = 0;
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2013-10-31 03:46:08 +08:00
|
|
|
# define CWD_STATE_FREE_ERR(state) do { \
|
|
|
|
DWORD last_error = GetLastError(); \
|
|
|
|
CWD_STATE_FREE(state); \
|
|
|
|
SetLastError(last_error); \
|
|
|
|
} while (0)
|
|
|
|
#else
|
|
|
|
# define CWD_STATE_FREE_ERR(state) CWD_STATE_FREE(state)
|
|
|
|
#endif
|
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2008-08-15 23:00:11 +08:00
|
|
|
|
2009-06-16 08:13:56 +08:00
|
|
|
#ifdef CTL_CODE
|
|
|
|
#undef CTL_CODE
|
|
|
|
#endif
|
|
|
|
#define CTL_CODE(DeviceType,Function,Method,Access) (((DeviceType) << 16) | ((Access) << 14) | ((Function) << 2) | (Method))
|
|
|
|
#define FILE_DEVICE_FILE_SYSTEM 0x00000009
|
|
|
|
#define METHOD_BUFFERED 0
|
|
|
|
#define FILE_ANY_ACCESS 0
|
|
|
|
#define FSCTL_GET_REPARSE_POINT CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 42, METHOD_BUFFERED, FILE_ANY_ACCESS)
|
|
|
|
#define MAXIMUM_REPARSE_DATA_BUFFER_SIZE ( 16 * 1024 )
|
|
|
|
|
|
|
|
typedef struct {
|
2014-09-22 14:53:15 +08:00
|
|
|
unsigned long ReparseTag;
|
2009-06-16 08:13:56 +08:00
|
|
|
unsigned short ReparseDataLength;
|
|
|
|
unsigned short Reserved;
|
|
|
|
union {
|
|
|
|
struct {
|
|
|
|
unsigned short SubstituteNameOffset;
|
|
|
|
unsigned short SubstituteNameLength;
|
|
|
|
unsigned short PrintNameOffset;
|
|
|
|
unsigned short PrintNameLength;
|
2014-09-22 14:53:15 +08:00
|
|
|
unsigned long Flags;
|
2009-06-16 08:13:56 +08:00
|
|
|
wchar_t ReparseTarget[1];
|
|
|
|
} SymbolicLinkReparseBuffer;
|
|
|
|
struct {
|
|
|
|
unsigned short SubstituteNameOffset;
|
|
|
|
unsigned short SubstituteNameLength;
|
|
|
|
unsigned short PrintNameOffset;
|
|
|
|
unsigned short PrintNameLength;
|
|
|
|
wchar_t ReparseTarget[1];
|
|
|
|
} MountPointReparseBuffer;
|
|
|
|
struct {
|
|
|
|
unsigned char ReparseTarget[1];
|
|
|
|
} GenericReparseBuffer;
|
|
|
|
};
|
2014-09-22 14:53:15 +08:00
|
|
|
} REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER;
|
2009-06-16 08:13:56 +08:00
|
|
|
|
2008-08-15 23:00:11 +08:00
|
|
|
#define SECS_BETWEEN_EPOCHS (__int64)11644473600
|
|
|
|
#define SECS_TO_100NS (__int64)10000000
|
2014-09-21 02:58:00 +08:00
|
|
|
static inline time_t FileTimeToUnixTime(const FILETIME *FileTime)
|
2008-08-15 23:00:11 +08:00
|
|
|
{
|
|
|
|
__int64 UnixTime;
|
|
|
|
SYSTEMTIME SystemTime;
|
2014-09-21 02:58:00 +08:00
|
|
|
FileTimeToSystemTime(FileTime, &SystemTime);
|
2008-08-15 23:00:11 +08:00
|
|
|
|
2014-09-21 02:58:00 +08:00
|
|
|
UnixTime = ((__int64)FileTime->dwHighDateTime << 32) +
|
|
|
|
FileTime->dwLowDateTime;
|
2008-08-15 23:00:11 +08:00
|
|
|
|
|
|
|
UnixTime -= (SECS_BETWEEN_EPOCHS * SECS_TO_100NS);
|
|
|
|
|
|
|
|
UnixTime /= SECS_TO_100NS; /* now convert to seconds */
|
|
|
|
|
|
|
|
if ((time_t)UnixTime != UnixTime) {
|
|
|
|
UnixTime = 0;
|
|
|
|
}
|
|
|
|
return (time_t)UnixTime;
|
|
|
|
}
|
|
|
|
|
2017-07-27 05:10:07 +08:00
|
|
|
CWD_API ssize_t php_sys_readlink(const char *link, char *target, size_t target_len){ /* {{{ */
|
2010-09-10 22:01:44 +08:00
|
|
|
HANDLE hFile;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
wchar_t *linkw = php_win32_ioutil_any_to_w(link), targetw[MAXPATHLEN];
|
2017-01-07 08:09:17 +08:00
|
|
|
size_t ret_len, targetw_len, offset = 0;
|
|
|
|
char *ret;
|
2010-09-10 22:01:44 +08:00
|
|
|
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
if (!linkw) {
|
2015-05-19 21:44:55 +08:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
if (!target_len) {
|
|
|
|
free(linkw);
|
2010-09-10 22:01:44 +08:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
hFile = CreateFileW(linkw, // file to open
|
2017-12-07 23:52:23 +08:00
|
|
|
0, // query possible attributes
|
|
|
|
PHP_WIN32_IOUTIL_DEFAULT_SHARE_MODE,
|
2010-09-10 22:01:44 +08:00
|
|
|
NULL, // default security
|
|
|
|
OPEN_EXISTING, // existing file only
|
|
|
|
FILE_FLAG_BACKUP_SEMANTICS, // normal file
|
|
|
|
NULL); // no attr. template
|
|
|
|
if( hFile == INVALID_HANDLE_VALUE) {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free(linkw);
|
|
|
|
return -1;
|
2010-09-10 22:01:44 +08:00
|
|
|
}
|
|
|
|
|
2015-05-19 21:44:55 +08:00
|
|
|
/* Despite MSDN has documented it won't to, the length returned by
|
|
|
|
GetFinalPathNameByHandleA includes the length of the
|
|
|
|
null terminator. This behavior is at least reproducible
|
|
|
|
with VS2012 and earlier, and seems not to be fixed till
|
|
|
|
now. Thus, correcting target_len so it's suddenly don't
|
|
|
|
overflown. */
|
2017-01-07 08:09:17 +08:00
|
|
|
targetw_len = GetFinalPathNameByHandleW(hFile, targetw, MAXPATHLEN, VOLUME_NAME_DOS);
|
|
|
|
if(targetw_len >= target_len || targetw_len >= MAXPATHLEN || targetw_len == 0) {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free(linkw);
|
|
|
|
CloseHandle(hFile);
|
2010-09-10 22:01:44 +08:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2017-01-07 08:09:17 +08:00
|
|
|
if(targetw_len > 4) {
|
|
|
|
/* Skip first 4 characters if they are "\\?\" */
|
|
|
|
if(targetw[0] == L'\\' && targetw[1] == L'\\' && targetw[2] == L'?' && targetw[3] == L'\\') {
|
|
|
|
offset = 4;
|
|
|
|
|
|
|
|
/* \\?\UNC\ */
|
|
|
|
if (targetw_len > 7 && targetw[4] == L'U' && targetw[5] == L'N' && targetw[6] == L'C') {
|
|
|
|
offset += 2;
|
|
|
|
targetw[offset] = L'\\';
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = php_win32_ioutil_conv_w_to_any(targetw + offset, targetw_len - offset, &ret_len);
|
|
|
|
if (!ret || ret_len >= MAXPATHLEN) {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
CloseHandle(hFile);
|
|
|
|
free(linkw);
|
2017-01-07 08:15:49 +08:00
|
|
|
free(ret);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2017-01-07 08:09:17 +08:00
|
|
|
memcpy(target, ret, ret_len + 1);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
|
2017-01-07 08:15:49 +08:00
|
|
|
free(ret);
|
2010-09-10 22:01:44 +08:00
|
|
|
CloseHandle(hFile);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free(linkw);
|
2010-09-10 22:01:44 +08:00
|
|
|
|
2017-07-27 05:10:07 +08:00
|
|
|
return (ssize_t)ret_len;
|
2010-09-10 22:01:44 +08:00
|
|
|
}
|
|
|
|
/* }}} */
|
|
|
|
|
2014-08-16 17:16:11 +08:00
|
|
|
CWD_API int php_sys_stat_ex(const char *path, zend_stat_t *buf, int lstat) /* {{{ */
|
2006-11-10 23:04:07 +08:00
|
|
|
{
|
|
|
|
WIN32_FILE_ATTRIBUTE_DATA data;
|
2014-10-02 18:15:34 +08:00
|
|
|
LARGE_INTEGER t;
|
2017-04-03 22:30:40 +08:00
|
|
|
size_t pathw_len = 0;
|
2016-07-30 23:30:49 +08:00
|
|
|
ALLOCA_FLAG(use_heap_large)
|
2017-04-03 22:30:40 +08:00
|
|
|
wchar_t *pathw = php_win32_ioutil_conv_any_to_w(path, PHP_WIN32_CP_IGNORE_LEN, &pathw_len);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
|
|
|
|
if (!pathw) {
|
|
|
|
return -1;
|
|
|
|
}
|
2006-11-10 23:04:07 +08:00
|
|
|
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
if (!GetFileAttributesExW(pathw, GetFileExInfoStandard, &data)) {
|
|
|
|
int ret;
|
|
|
|
#if ZEND_ENABLE_ZVAL_LONG64
|
|
|
|
ret = _wstat64(pathw, buf);
|
|
|
|
#else
|
2016-07-07 21:34:46 +08:00
|
|
|
ret = _wstat(pathw, (struct _stat32 *)buf);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#endif
|
|
|
|
free(pathw);
|
|
|
|
|
|
|
|
return ret;
|
2006-11-10 23:04:07 +08:00
|
|
|
}
|
|
|
|
|
2017-04-03 22:30:40 +08:00
|
|
|
if (pathw_len >= 1 && pathw[1] == L':') {
|
|
|
|
if (pathw[0] >= L'A' && pathw[0] <= L'Z') {
|
|
|
|
buf->st_dev = buf->st_rdev = pathw[0] - L'A';
|
2006-11-10 23:04:07 +08:00
|
|
|
} else {
|
2017-04-03 22:30:40 +08:00
|
|
|
buf->st_dev = buf->st_rdev = pathw[0] - L'a';
|
2006-11-10 23:04:07 +08:00
|
|
|
}
|
2017-04-03 22:30:40 +08:00
|
|
|
} else if (PHP_WIN32_IOUTIL_IS_UNC(pathw, pathw_len)) {
|
2010-09-13 18:58:18 +08:00
|
|
|
buf->st_dev = buf->st_rdev = 0;
|
2006-11-10 23:04:07 +08:00
|
|
|
} else {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
wchar_t cur_path[MAXPATHLEN+1];
|
2016-09-02 03:31:40 +08:00
|
|
|
|
|
|
|
if (NULL != _wgetcwd(cur_path, sizeof(cur_path)/sizeof(wchar_t))) {
|
|
|
|
if (cur_path[1] == L':') {
|
|
|
|
if (pathw[0] >= L'A' && pathw[0] <= L'Z') {
|
|
|
|
buf->st_dev = buf->st_rdev = pathw[0] - L'A';
|
2006-11-10 23:04:07 +08:00
|
|
|
} else {
|
2016-09-02 03:31:40 +08:00
|
|
|
buf->st_dev = buf->st_rdev = pathw[0] - L'a';
|
2006-11-10 23:04:07 +08:00
|
|
|
}
|
|
|
|
} else {
|
2016-09-02 03:31:40 +08:00
|
|
|
buf->st_dev = buf->st_rdev = -1;
|
2006-11-10 23:04:07 +08:00
|
|
|
}
|
2016-09-02 03:31:40 +08:00
|
|
|
} else {
|
|
|
|
buf->st_dev = buf->st_rdev = -1;
|
2010-08-27 18:09:52 +08:00
|
|
|
}
|
2006-11-10 23:04:07 +08:00
|
|
|
}
|
2010-09-01 17:49:53 +08:00
|
|
|
|
2006-11-10 23:04:07 +08:00
|
|
|
buf->st_uid = buf->st_gid = buf->st_ino = 0;
|
2010-09-01 17:49:53 +08:00
|
|
|
|
|
|
|
if (lstat && data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) {
|
|
|
|
/* File is a reparse point. Get the target */
|
|
|
|
HANDLE hLink = NULL;
|
|
|
|
REPARSE_DATA_BUFFER * pbuffer;
|
2014-11-18 18:49:06 +08:00
|
|
|
DWORD retlength = 0;
|
2010-09-01 17:49:53 +08:00
|
|
|
|
2017-12-07 23:52:23 +08:00
|
|
|
hLink = CreateFileW(pathw,
|
|
|
|
FILE_READ_ATTRIBUTES,
|
|
|
|
PHP_WIN32_IOUTIL_DEFAULT_SHARE_MODE,
|
|
|
|
NULL,
|
|
|
|
OPEN_EXISTING,
|
|
|
|
FILE_FLAG_OPEN_REPARSE_POINT|FILE_FLAG_BACKUP_SEMANTICS,
|
|
|
|
NULL);
|
2010-09-01 17:49:53 +08:00
|
|
|
if(hLink == INVALID_HANDLE_VALUE) {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free(pathw);
|
2010-09-01 17:49:53 +08:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2013-10-24 23:48:09 +08:00
|
|
|
pbuffer = (REPARSE_DATA_BUFFER *)do_alloca(MAXIMUM_REPARSE_DATA_BUFFER_SIZE, use_heap_large);
|
2010-09-01 17:49:53 +08:00
|
|
|
if(!DeviceIoControl(hLink, FSCTL_GET_REPARSE_POINT, NULL, 0, pbuffer, MAXIMUM_REPARSE_DATA_BUFFER_SIZE, &retlength, NULL)) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
2010-09-01 17:49:53 +08:00
|
|
|
CloseHandle(hLink);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free(pathw);
|
2010-09-01 17:49:53 +08:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
CloseHandle(hLink);
|
|
|
|
|
|
|
|
if(pbuffer->ReparseTag == IO_REPARSE_TAG_SYMLINK) {
|
|
|
|
buf->st_mode = S_IFLNK;
|
|
|
|
buf->st_mode |= (data.dwFileAttributes & FILE_ATTRIBUTE_READONLY) ? (S_IREAD|(S_IREAD>>3)|(S_IREAD>>6)) : (S_IREAD|(S_IREAD>>3)|(S_IREAD>>6)|S_IWRITE|(S_IWRITE>>3)|(S_IWRITE>>6));
|
|
|
|
}
|
|
|
|
|
|
|
|
#if 0 /* Not used yet */
|
|
|
|
else if(pbuffer->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT) {
|
|
|
|
buf->st_mode |=;
|
|
|
|
}
|
|
|
|
#endif
|
2013-10-24 23:48:09 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
2010-09-01 17:49:53 +08:00
|
|
|
} else {
|
|
|
|
buf->st_mode = (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ? (S_IFDIR|S_IEXEC|(S_IEXEC>>3)|(S_IEXEC>>6)) : S_IFREG;
|
|
|
|
buf->st_mode |= (data.dwFileAttributes & FILE_ATTRIBUTE_READONLY) ? (S_IREAD|(S_IREAD>>3)|(S_IREAD>>6)) : (S_IREAD|(S_IREAD>>3)|(S_IREAD>>6)|S_IWRITE|(S_IWRITE>>3)|(S_IWRITE>>6));
|
|
|
|
}
|
|
|
|
|
2006-11-10 23:04:07 +08:00
|
|
|
if ((data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) {
|
2017-04-03 22:30:40 +08:00
|
|
|
if (pathw_len >= 4 && pathw[pathw_len-4] == L'.') {
|
|
|
|
if (_wcsnicmp(pathw+pathw_len-3, L"exe", 3) == 0 ||
|
|
|
|
_wcsnicmp(pathw+pathw_len-3, L"com", 3) == 0 ||
|
|
|
|
_wcsnicmp(pathw+pathw_len-3, L"bat", 3) == 0 ||
|
|
|
|
_wcsnicmp(pathw+pathw_len-3, L"cmd", 3) == 0) {
|
2006-11-10 23:04:07 +08:00
|
|
|
buf->st_mode |= (S_IEXEC|(S_IEXEC>>3)|(S_IEXEC>>6));
|
|
|
|
}
|
|
|
|
}
|
2008-08-15 23:00:11 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
buf->st_nlink = 1;
|
2014-10-02 18:15:34 +08:00
|
|
|
t.HighPart = data.nFileSizeHigh;
|
|
|
|
t.LowPart = data.nFileSizeLow;
|
|
|
|
/* It's an overflow on 32 bit, however it won't fix as long
|
|
|
|
as zend_long is 32 bit. */
|
|
|
|
buf->st_size = (zend_long)t.QuadPart;
|
2014-09-21 02:58:00 +08:00
|
|
|
buf->st_atime = FileTimeToUnixTime(&data.ftLastAccessTime);
|
|
|
|
buf->st_ctime = FileTimeToUnixTime(&data.ftCreationTime);
|
|
|
|
buf->st_mtime = FileTimeToUnixTime(&data.ftLastWriteTime);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
|
|
|
|
free(pathw);
|
|
|
|
|
2006-11-10 23:04:07 +08:00
|
|
|
return 0;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2006-11-10 23:04:07 +08:00
|
|
|
#endif
|
|
|
|
|
2007-08-10 17:09:46 +08:00
|
|
|
static int php_is_dir_ok(const cwd_state *state) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2014-08-16 17:16:11 +08:00
|
|
|
zend_stat_t buf;
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2006-11-10 23:04:07 +08:00
|
|
|
if (php_sys_stat(state->cwd, &buf) == 0 && S_ISDIR(buf.st_mode))
|
2000-09-04 02:18:13 +08:00
|
|
|
return (0);
|
|
|
|
|
|
|
|
return (1);
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2007-08-10 17:09:46 +08:00
|
|
|
static int php_is_file_ok(const cwd_state *state) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2014-08-16 17:16:11 +08:00
|
|
|
zend_stat_t buf;
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2006-11-10 23:04:07 +08:00
|
|
|
if (php_sys_stat(state->cwd, &buf) == 0 && S_ISREG(buf.st_mode))
|
2000-09-04 02:18:13 +08:00
|
|
|
return (0);
|
|
|
|
|
|
|
|
return (1);
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
static void cwd_globals_ctor(virtual_cwd_globals *cwd_g) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2010-06-08 21:27:30 +08:00
|
|
|
CWD_STATE_COPY(&cwd_g->cwd, &main_cwd_state);
|
|
|
|
cwd_g->realpath_cache_size = 0;
|
|
|
|
cwd_g->realpath_cache_size_limit = REALPATH_CACHE_SIZE;
|
|
|
|
cwd_g->realpath_cache_ttl = REALPATH_CACHE_TTL;
|
|
|
|
memset(cwd_g->realpath_cache, 0, sizeof(cwd_g->realpath_cache));
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
static void cwd_globals_dtor(virtual_cwd_globals *cwd_g) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2014-12-14 06:06:14 +08:00
|
|
|
realpath_cache_clean();
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2017-08-14 06:44:19 +08:00
|
|
|
void virtual_cwd_main_cwd_init(uint8_t reinit) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
char cwd[MAXPATHLEN];
|
|
|
|
char *result;
|
|
|
|
|
2017-08-14 06:44:19 +08:00
|
|
|
if (reinit) {
|
|
|
|
free(main_cwd_state.cwd);
|
|
|
|
}
|
2016-11-12 18:20:01 +08:00
|
|
|
|
2015-11-05 05:29:27 +08:00
|
|
|
#ifdef ZEND_WIN32
|
|
|
|
ZeroMemory(&cwd, sizeof(cwd));
|
2017-08-14 06:44:19 +08:00
|
|
|
result = php_win32_ioutil_getcwd(cwd, sizeof(cwd));
|
|
|
|
#else
|
2010-09-10 22:02:19 +08:00
|
|
|
result = getcwd(cwd, sizeof(cwd));
|
2005-07-07 15:10:50 +08:00
|
|
|
#endif
|
2016-11-12 18:20:01 +08:00
|
|
|
|
2000-09-04 02:18:13 +08:00
|
|
|
if (!result) {
|
|
|
|
cwd[0] = '\0';
|
|
|
|
}
|
2008-08-15 23:00:11 +08:00
|
|
|
|
2017-07-27 05:10:07 +08:00
|
|
|
main_cwd_state.cwd_length = strlen(cwd);
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2008-08-15 23:00:11 +08:00
|
|
|
if (main_cwd_state.cwd_length >= 2 && cwd[1] == ':') {
|
|
|
|
cwd[0] = toupper(cwd[0]);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
main_cwd_state.cwd = strdup(cwd);
|
2017-08-14 06:44:19 +08:00
|
|
|
}
|
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2017-08-14 06:44:19 +08:00
|
|
|
CWD_API void virtual_cwd_startup(void) /* {{{ */
|
|
|
|
{
|
|
|
|
virtual_cwd_main_cwd_init(0);
|
2000-09-04 02:18:13 +08:00
|
|
|
#ifdef ZTS
|
2001-07-27 18:08:26 +08:00
|
|
|
ts_allocate_id(&cwd_globals_id, sizeof(virtual_cwd_globals), (ts_allocate_ctor) cwd_globals_ctor, (ts_allocate_dtor) cwd_globals_dtor);
|
2000-09-04 02:18:13 +08:00
|
|
|
#else
|
2014-12-14 06:06:14 +08:00
|
|
|
cwd_globals_ctor(&cwd_globals);
|
2000-09-04 02:18:13 +08:00
|
|
|
#endif
|
|
|
|
|
2016-11-12 18:20:01 +08:00
|
|
|
#if (defined(ZEND_WIN32)) && defined(ZTS)
|
2000-09-04 02:18:13 +08:00
|
|
|
cwd_mutex = tsrm_mutex_alloc();
|
|
|
|
#endif
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2007-08-10 17:09:46 +08:00
|
|
|
CWD_API void virtual_cwd_shutdown(void) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
#ifndef ZTS
|
2014-12-14 06:06:14 +08:00
|
|
|
cwd_globals_dtor(&cwd_globals);
|
2000-09-04 02:18:13 +08:00
|
|
|
#endif
|
2016-11-12 18:20:01 +08:00
|
|
|
#if (defined(ZEND_WIN32)) && defined(ZTS)
|
2000-09-04 02:18:13 +08:00
|
|
|
tsrm_mutex_free(cwd_mutex);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
free(main_cwd_state.cwd); /* Don't use CWD_STATE_FREE because the non global states will probably use emalloc()/efree() */
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_cwd_activate(void) /* {{{ */
|
2013-10-17 16:40:43 +08:00
|
|
|
{
|
|
|
|
if (CWDG(cwd).cwd == NULL) {
|
|
|
|
CWD_STATE_COPY(&CWDG(cwd), &main_cwd_state);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
/* }}} */
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_cwd_deactivate(void) /* {{{ */
|
2013-10-17 16:40:43 +08:00
|
|
|
{
|
|
|
|
if (CWDG(cwd).cwd != NULL) {
|
|
|
|
CWD_STATE_FREE(&CWDG(cwd));
|
2013-11-01 00:39:17 +08:00
|
|
|
CWDG(cwd).cwd = NULL;
|
2013-10-17 16:40:43 +08:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
/* }}} */
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API char *virtual_getcwd_ex(size_t *length) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state *state;
|
|
|
|
|
|
|
|
state = &CWDG(cwd);
|
|
|
|
|
|
|
|
if (state->cwd_length == 0) {
|
|
|
|
char *retval;
|
|
|
|
|
|
|
|
*length = 1;
|
2013-10-17 16:40:43 +08:00
|
|
|
retval = (char *) emalloc(2);
|
2000-09-04 02:18:13 +08:00
|
|
|
retval[0] = DEFAULT_SLASH;
|
2010-09-10 22:02:19 +08:00
|
|
|
retval[1] = '\0';
|
2000-09-04 02:18:13 +08:00
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2000-09-04 02:18:13 +08:00
|
|
|
/* If we have something like C: */
|
|
|
|
if (state->cwd_length == 2 && state->cwd[state->cwd_length-1] == ':') {
|
|
|
|
char *retval;
|
|
|
|
|
|
|
|
*length = state->cwd_length+1;
|
2013-10-17 16:40:43 +08:00
|
|
|
retval = (char *) emalloc(*length+1);
|
2000-09-04 02:18:13 +08:00
|
|
|
memcpy(retval, state->cwd, *length);
|
2008-08-15 23:00:11 +08:00
|
|
|
retval[0] = toupper(retval[0]);
|
2000-09-04 02:18:13 +08:00
|
|
|
retval[*length-1] = DEFAULT_SLASH;
|
|
|
|
retval[*length] = '\0';
|
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
#endif
|
2015-06-14 00:44:56 +08:00
|
|
|
if (!state->cwd) {
|
|
|
|
*length = 0;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2000-09-04 02:18:13 +08:00
|
|
|
*length = state->cwd_length;
|
2013-10-17 16:40:43 +08:00
|
|
|
return estrdup(state->cwd);
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
|
|
|
/* Same semantics as UNIX getcwd() */
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API char *virtual_getcwd(char *buf, size_t size) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
size_t length;
|
|
|
|
char *cwd;
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
cwd = virtual_getcwd_ex(&length);
|
2000-09-04 02:18:13 +08:00
|
|
|
|
|
|
|
if (buf == NULL) {
|
|
|
|
return cwd;
|
|
|
|
}
|
|
|
|
if (length > size-1) {
|
2013-10-17 16:40:43 +08:00
|
|
|
efree(cwd);
|
2000-09-04 02:18:13 +08:00
|
|
|
errno = ERANGE; /* Is this OK? */
|
|
|
|
return NULL;
|
|
|
|
}
|
2015-06-14 00:44:56 +08:00
|
|
|
if (!cwd) {
|
|
|
|
return NULL;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
memcpy(buf, cwd, length+1);
|
2013-10-17 16:40:43 +08:00
|
|
|
efree(cwd);
|
2000-09-04 02:18:13 +08:00
|
|
|
return buf;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2016-12-22 21:56:47 +08:00
|
|
|
static inline zend_ulong realpath_cache_key(const char *path, size_t path_len) /* {{{ */
|
2009-10-20 07:32:07 +08:00
|
|
|
{
|
2014-08-26 01:24:55 +08:00
|
|
|
register zend_ulong h;
|
2017-04-27 21:39:32 +08:00
|
|
|
size_t bucket_key_len;
|
|
|
|
char *bucket_key_start = tsrm_win32_get_path_sid_key(path, path_len, &bucket_key_len);
|
2010-01-28 00:43:16 +08:00
|
|
|
char *bucket_key = (char *)bucket_key_start;
|
2016-03-10 14:24:13 +08:00
|
|
|
const char *e;
|
2009-10-20 07:32:07 +08:00
|
|
|
|
|
|
|
if (!bucket_key) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-04-27 21:39:32 +08:00
|
|
|
e = bucket_key + bucket_key_len;
|
2014-08-26 02:57:25 +08:00
|
|
|
for (h = Z_UL(2166136261); bucket_key < e;) {
|
|
|
|
h *= Z_UL(16777619);
|
2009-10-20 07:32:07 +08:00
|
|
|
h ^= *bucket_key++;
|
|
|
|
}
|
2017-04-27 22:03:12 +08:00
|
|
|
if (bucket_key_start != path) {
|
|
|
|
HeapFree(GetProcessHeap(), 0, (LPVOID)bucket_key_start);
|
|
|
|
}
|
2009-10-20 07:32:07 +08:00
|
|
|
return h;
|
|
|
|
}
|
|
|
|
/* }}} */
|
|
|
|
#else
|
2016-12-22 21:56:47 +08:00
|
|
|
static inline zend_ulong realpath_cache_key(const char *path, size_t path_len) /* {{{ */
|
2004-10-05 08:42:25 +08:00
|
|
|
{
|
2014-08-26 01:24:55 +08:00
|
|
|
register zend_ulong h;
|
2007-01-21 00:22:51 +08:00
|
|
|
const char *e = path + path_len;
|
|
|
|
|
2014-08-26 02:57:25 +08:00
|
|
|
for (h = Z_UL(2166136261); path < e;) {
|
|
|
|
h *= Z_UL(16777619);
|
2007-01-21 00:22:51 +08:00
|
|
|
h ^= *path++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return h;
|
2004-10-05 08:42:25 +08:00
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2015-07-05 00:55:22 +08:00
|
|
|
#endif /* defined(ZEND_WIN32) */
|
2004-10-05 08:42:25 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API void realpath_cache_clean(void) /* {{{ */
|
2007-01-22 17:31:46 +08:00
|
|
|
{
|
2014-12-13 02:57:34 +08:00
|
|
|
uint32_t i;
|
2007-01-22 17:31:46 +08:00
|
|
|
|
|
|
|
for (i = 0; i < sizeof(CWDG(realpath_cache))/sizeof(CWDG(realpath_cache)[0]); i++) {
|
|
|
|
realpath_cache_bucket *p = CWDG(realpath_cache)[i];
|
|
|
|
while (p != NULL) {
|
|
|
|
realpath_cache_bucket *r = p;
|
|
|
|
p = p->next;
|
|
|
|
free(r);
|
|
|
|
}
|
|
|
|
CWDG(realpath_cache)[i] = NULL;
|
|
|
|
}
|
|
|
|
CWDG(realpath_cache_size) = 0;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2007-01-22 17:31:46 +08:00
|
|
|
|
2016-12-22 21:56:47 +08:00
|
|
|
CWD_API void realpath_cache_del(const char *path, size_t path_len) /* {{{ */
|
2007-01-22 17:31:46 +08:00
|
|
|
{
|
2014-12-14 06:06:14 +08:00
|
|
|
zend_ulong key = realpath_cache_key(path, path_len);
|
2014-08-26 01:24:55 +08:00
|
|
|
zend_ulong n = key % (sizeof(CWDG(realpath_cache)) / sizeof(CWDG(realpath_cache)[0]));
|
2007-01-22 17:31:46 +08:00
|
|
|
realpath_cache_bucket **bucket = &CWDG(realpath_cache)[n];
|
|
|
|
|
|
|
|
while (*bucket != NULL) {
|
|
|
|
if (key == (*bucket)->key && path_len == (*bucket)->path_len &&
|
2010-08-27 18:09:52 +08:00
|
|
|
memcmp(path, (*bucket)->path, path_len) == 0) {
|
2007-01-22 17:31:46 +08:00
|
|
|
realpath_cache_bucket *r = *bucket;
|
|
|
|
*bucket = (*bucket)->next;
|
2015-01-03 17:22:58 +08:00
|
|
|
|
2011-05-20 07:09:58 +08:00
|
|
|
/* if the pointers match then only subtract the length of the path */
|
2011-05-20 07:18:55 +08:00
|
|
|
if(r->path == r->realpath) {
|
|
|
|
CWDG(realpath_cache_size) -= sizeof(realpath_cache_bucket) + r->path_len + 1;
|
|
|
|
} else {
|
|
|
|
CWDG(realpath_cache_size) -= sizeof(realpath_cache_bucket) + r->path_len + 1 + r->realpath_len + 1;
|
|
|
|
}
|
2015-01-03 17:22:58 +08:00
|
|
|
|
2007-01-22 17:31:46 +08:00
|
|
|
free(r);
|
|
|
|
return;
|
|
|
|
} else {
|
|
|
|
bucket = &(*bucket)->next;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2007-01-22 17:31:46 +08:00
|
|
|
|
2017-10-11 14:58:59 +08:00
|
|
|
static inline void realpath_cache_add(const char *path, size_t path_len, const char *realpath, size_t realpath_len, int is_dir, time_t t) /* {{{ */
|
2004-10-05 08:42:25 +08:00
|
|
|
{
|
2014-08-26 01:24:55 +08:00
|
|
|
zend_long size = sizeof(realpath_cache_bucket) + path_len + 1;
|
2008-08-12 16:01:24 +08:00
|
|
|
int same = 1;
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
if (realpath_len != path_len ||
|
2010-08-27 18:09:52 +08:00
|
|
|
memcmp(path, realpath, path_len) != 0) {
|
2008-08-12 16:01:24 +08:00
|
|
|
size += realpath_len + 1;
|
|
|
|
same = 0;
|
|
|
|
}
|
|
|
|
|
2004-10-05 08:42:25 +08:00
|
|
|
if (CWDG(realpath_cache_size) + size <= CWDG(realpath_cache_size_limit)) {
|
|
|
|
realpath_cache_bucket *bucket = malloc(size);
|
2014-08-26 01:24:55 +08:00
|
|
|
zend_ulong n;
|
2009-10-20 07:32:07 +08:00
|
|
|
|
2011-01-10 08:43:08 +08:00
|
|
|
if (bucket == NULL) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2004-10-05 08:42:25 +08:00
|
|
|
bucket->key = realpath_cache_key(path, path_len);
|
|
|
|
bucket->path = (char*)bucket + sizeof(realpath_cache_bucket);
|
|
|
|
memcpy(bucket->path, path, path_len+1);
|
|
|
|
bucket->path_len = path_len;
|
2008-08-12 16:01:24 +08:00
|
|
|
if (same) {
|
|
|
|
bucket->realpath = bucket->path;
|
|
|
|
} else {
|
|
|
|
bucket->realpath = bucket->path + (path_len + 1);
|
|
|
|
memcpy(bucket->realpath, realpath, realpath_len+1);
|
|
|
|
}
|
2004-10-05 08:42:25 +08:00
|
|
|
bucket->realpath_len = realpath_len;
|
2016-11-13 23:48:36 +08:00
|
|
|
bucket->is_dir = is_dir > 0;
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2009-06-16 08:07:05 +08:00
|
|
|
bucket->is_rvalid = 0;
|
|
|
|
bucket->is_readable = 0;
|
|
|
|
bucket->is_wvalid = 0;
|
|
|
|
bucket->is_writable = 0;
|
|
|
|
#endif
|
2004-10-05 08:42:25 +08:00
|
|
|
bucket->expires = t + CWDG(realpath_cache_ttl);
|
|
|
|
n = bucket->key % (sizeof(CWDG(realpath_cache)) / sizeof(CWDG(realpath_cache)[0]));
|
|
|
|
bucket->next = CWDG(realpath_cache)[n];
|
|
|
|
CWDG(realpath_cache)[n] = bucket;
|
2007-01-21 00:22:51 +08:00
|
|
|
CWDG(realpath_cache_size) += size;
|
2004-10-05 08:42:25 +08:00
|
|
|
}
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2004-10-05 08:42:25 +08:00
|
|
|
|
2016-12-22 21:56:47 +08:00
|
|
|
static inline realpath_cache_bucket* realpath_cache_find(const char *path, size_t path_len, time_t t) /* {{{ */
|
2004-10-05 08:42:25 +08:00
|
|
|
{
|
2014-08-26 01:24:55 +08:00
|
|
|
zend_ulong key = realpath_cache_key(path, path_len);
|
|
|
|
zend_ulong n = key % (sizeof(CWDG(realpath_cache)) / sizeof(CWDG(realpath_cache)[0]));
|
2004-10-05 08:42:25 +08:00
|
|
|
realpath_cache_bucket **bucket = &CWDG(realpath_cache)[n];
|
|
|
|
|
|
|
|
while (*bucket != NULL) {
|
|
|
|
if (CWDG(realpath_cache_ttl) && (*bucket)->expires < t) {
|
|
|
|
realpath_cache_bucket *r = *bucket;
|
2006-01-16 17:09:37 +08:00
|
|
|
*bucket = (*bucket)->next;
|
2011-05-20 07:09:58 +08:00
|
|
|
|
2015-01-03 17:22:58 +08:00
|
|
|
/* if the pointers match then only subtract the length of the path */
|
2011-05-20 07:18:55 +08:00
|
|
|
if(r->path == r->realpath) {
|
|
|
|
CWDG(realpath_cache_size) -= sizeof(realpath_cache_bucket) + r->path_len + 1;
|
|
|
|
} else {
|
|
|
|
CWDG(realpath_cache_size) -= sizeof(realpath_cache_bucket) + r->path_len + 1 + r->realpath_len + 1;
|
|
|
|
}
|
2006-01-16 17:09:37 +08:00
|
|
|
free(r);
|
2004-10-05 08:42:25 +08:00
|
|
|
} else if (key == (*bucket)->key && path_len == (*bucket)->path_len &&
|
2010-08-27 18:09:52 +08:00
|
|
|
memcmp(path, (*bucket)->path, path_len) == 0) {
|
2004-10-05 08:42:25 +08:00
|
|
|
return *bucket;
|
|
|
|
} else {
|
2006-01-16 17:09:37 +08:00
|
|
|
bucket = &(*bucket)->next;
|
2004-10-05 08:42:25 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2004-10-05 08:42:25 +08:00
|
|
|
|
2016-12-22 21:56:47 +08:00
|
|
|
CWD_API realpath_cache_bucket* realpath_cache_lookup(const char *path, size_t path_len, time_t t) /* {{{ */
|
2009-06-16 08:07:05 +08:00
|
|
|
{
|
2014-12-14 06:06:14 +08:00
|
|
|
return realpath_cache_find(path, path_len, t);
|
2009-06-16 08:07:05 +08:00
|
|
|
}
|
|
|
|
/* }}} */
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API zend_long realpath_cache_size(void)
|
2009-12-08 09:35:08 +08:00
|
|
|
{
|
|
|
|
return CWDG(realpath_cache_size);
|
|
|
|
}
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API zend_long realpath_cache_max_buckets(void)
|
2009-12-08 09:35:08 +08:00
|
|
|
{
|
|
|
|
return (sizeof(CWDG(realpath_cache)) / sizeof(CWDG(realpath_cache)[0]));
|
|
|
|
}
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API realpath_cache_bucket** realpath_cache_get_buckets(void)
|
2009-12-08 09:35:08 +08:00
|
|
|
{
|
|
|
|
return CWDG(realpath_cache);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2008-11-13 09:35:21 +08:00
|
|
|
#undef LINK_MAX
|
2008-08-12 16:01:24 +08:00
|
|
|
#define LINK_MAX 32
|
|
|
|
|
2017-07-27 05:10:07 +08:00
|
|
|
static size_t tsrm_realpath_r(char *path, size_t start, size_t len, int *ll, time_t *t, int use_realpath, int is_dir, int *link_is_dir) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2017-07-27 05:10:07 +08:00
|
|
|
size_t i, j;
|
|
|
|
int directory = 0, save;
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
WIN32_FIND_DATAW dataw;
|
|
|
|
HANDLE hFind = INVALID_HANDLE_VALUE;
|
2013-10-24 20:12:37 +08:00
|
|
|
ALLOCA_FLAG(use_heap_large)
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
wchar_t *pathw = NULL;
|
|
|
|
#define FREE_PATHW() \
|
|
|
|
do { free(pathw); } while(0);
|
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
#else
|
2014-08-16 17:16:11 +08:00
|
|
|
zend_stat_t st;
|
2007-10-23 13:57:25 +08:00
|
|
|
#endif
|
2008-08-12 16:01:24 +08:00
|
|
|
realpath_cache_bucket *bucket;
|
|
|
|
char *tmp;
|
2013-10-24 20:12:37 +08:00
|
|
|
ALLOCA_FLAG(use_heap)
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
while (1) {
|
|
|
|
if (len <= start) {
|
2012-01-30 18:08:11 +08:00
|
|
|
if (link_is_dir) {
|
|
|
|
*link_is_dir = 1;
|
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
return start;
|
|
|
|
}
|
2006-09-20 15:35:59 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
i = len;
|
|
|
|
while (i > start && !IS_SLASH(path[i-1])) {
|
|
|
|
i--;
|
|
|
|
}
|
2017-07-28 05:38:04 +08:00
|
|
|
assert(i < MAXPATHLEN);
|
2006-09-20 15:35:59 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
if (i == len ||
|
2017-07-27 05:10:07 +08:00
|
|
|
(i + 1 == len && path[i] == '.')) {
|
2008-08-12 16:01:24 +08:00
|
|
|
/* remove double slashes and '.' */
|
2019-01-19 09:34:59 +08:00
|
|
|
len = EXPECTED(i > 0) ? i - 1 : 0;
|
2008-08-12 16:01:24 +08:00
|
|
|
is_dir = 1;
|
|
|
|
continue;
|
2017-07-28 05:38:04 +08:00
|
|
|
} else if (i + 2 == len && path[i] == '.' && path[i+1] == '.') {
|
2008-08-12 16:01:24 +08:00
|
|
|
/* remove '..' and previous directory */
|
2012-01-30 18:08:11 +08:00
|
|
|
is_dir = 1;
|
|
|
|
if (link_is_dir) {
|
|
|
|
*link_is_dir = 1;
|
|
|
|
}
|
2017-07-27 05:10:07 +08:00
|
|
|
if (i <= start + 1) {
|
2008-08-12 16:01:24 +08:00
|
|
|
return start ? start : len;
|
2007-04-04 04:20:56 +08:00
|
|
|
}
|
2014-12-14 06:06:14 +08:00
|
|
|
j = tsrm_realpath_r(path, start, i-1, ll, t, use_realpath, 1, NULL);
|
2017-07-28 05:38:04 +08:00
|
|
|
if (j > start && j != (size_t)-1) {
|
2008-08-12 16:01:24 +08:00
|
|
|
j--;
|
2017-07-28 05:38:04 +08:00
|
|
|
assert(i < MAXPATHLEN);
|
2008-08-12 16:01:24 +08:00
|
|
|
while (j > start && !IS_SLASH(path[j])) {
|
|
|
|
j--;
|
|
|
|
}
|
2017-07-28 05:38:04 +08:00
|
|
|
assert(i < MAXPATHLEN);
|
2008-08-12 16:01:24 +08:00
|
|
|
if (!start) {
|
|
|
|
/* leading '..' must not be removed in case of relative path */
|
|
|
|
if (j == 0 && path[0] == '.' && path[1] == '.' &&
|
2010-08-27 18:09:52 +08:00
|
|
|
IS_SLASH(path[2])) {
|
2008-08-12 16:01:24 +08:00
|
|
|
path[3] = '.';
|
|
|
|
path[4] = '.';
|
|
|
|
path[5] = DEFAULT_SLASH;
|
|
|
|
j = 5;
|
2010-09-10 22:02:19 +08:00
|
|
|
} else if (j > 0 &&
|
2010-08-27 18:09:52 +08:00
|
|
|
path[j+1] == '.' && path[j+2] == '.' &&
|
|
|
|
IS_SLASH(path[j+3])) {
|
2008-08-12 16:01:24 +08:00
|
|
|
j += 4;
|
|
|
|
path[j++] = '.';
|
|
|
|
path[j++] = '.';
|
|
|
|
path[j] = DEFAULT_SLASH;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else if (!start && !j) {
|
|
|
|
/* leading '..' must not be removed in case of relative path */
|
|
|
|
path[0] = '.';
|
|
|
|
path[1] = '.';
|
|
|
|
path[2] = DEFAULT_SLASH;
|
|
|
|
j = 2;
|
2007-04-04 04:20:56 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
return j;
|
2004-10-05 08:42:25 +08:00
|
|
|
}
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
path[len] = 0;
|
2006-09-20 15:35:59 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
save = (use_realpath != CWD_EXPAND);
|
|
|
|
|
|
|
|
if (start && save && CWDG(realpath_cache_size_limit)) {
|
|
|
|
/* cache lookup for absolute path */
|
|
|
|
if (!*t) {
|
|
|
|
*t = time(0);
|
2004-10-05 08:42:25 +08:00
|
|
|
}
|
2014-12-14 06:06:14 +08:00
|
|
|
if ((bucket = realpath_cache_find(path, len, *t)) != NULL) {
|
2010-08-27 18:09:52 +08:00
|
|
|
if (is_dir && !bucket->is_dir) {
|
2008-08-12 16:01:24 +08:00
|
|
|
/* not a directory */
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2010-08-27 18:09:52 +08:00
|
|
|
} else {
|
|
|
|
if (link_is_dir) {
|
|
|
|
*link_is_dir = bucket->is_dir;
|
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
memcpy(path, bucket->realpath, bucket->realpath_len + 1);
|
2010-08-27 18:09:52 +08:00
|
|
|
return bucket->realpath_len;
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
2010-08-27 18:09:52 +08:00
|
|
|
}
|
2004-10-05 08:42:25 +08:00
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
if (save) {
|
|
|
|
pathw = php_win32_ioutil_any_to_w(path);
|
|
|
|
if (!pathw) {
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2006-11-11 01:23:58 +08:00
|
|
|
}
|
2017-04-08 03:12:52 +08:00
|
|
|
hFind = FindFirstFileExW(pathw, FindExInfoBasic, &dataw, FindExSearchNameMatch, NULL, 0);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
if (INVALID_HANDLE_VALUE == hFind) {
|
|
|
|
if (use_realpath == CWD_REALPATH) {
|
|
|
|
/* file not found */
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
}
|
|
|
|
/* continue resolution anyway but don't save result in the cache */
|
|
|
|
save = 0;
|
|
|
|
} else {
|
|
|
|
FindClose(hFind);
|
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
2009-06-16 08:13:56 +08:00
|
|
|
|
2013-10-24 20:12:37 +08:00
|
|
|
tmp = do_alloca(len+1, use_heap);
|
2008-08-12 16:01:24 +08:00
|
|
|
memcpy(tmp, path, len+1);
|
2009-06-16 08:13:56 +08:00
|
|
|
|
2020-07-27 23:50:41 +08:00
|
|
|
retry:
|
2010-09-10 22:02:19 +08:00
|
|
|
if(save &&
|
2011-01-10 08:59:19 +08:00
|
|
|
!(IS_UNC_PATH(path, len) && len >= 3 && path[2] != '?') &&
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
(dataw.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
|
|
|
|
) {
|
2009-06-16 08:13:56 +08:00
|
|
|
/* File is a reparse point. Get the target */
|
|
|
|
HANDLE hLink = NULL;
|
|
|
|
REPARSE_DATA_BUFFER * pbuffer;
|
2014-11-18 18:49:06 +08:00
|
|
|
DWORD retlength = 0;
|
2017-07-27 05:10:07 +08:00
|
|
|
size_t bufindex = 0;
|
|
|
|
uint8_t isabsolute = 0;
|
2009-06-16 08:13:56 +08:00
|
|
|
wchar_t * reparsetarget;
|
2009-09-02 01:50:59 +08:00
|
|
|
BOOL isVolume = FALSE;
|
2017-01-21 09:17:06 +08:00
|
|
|
#if VIRTUAL_CWD_DEBUG
|
|
|
|
char *printname = NULL;
|
|
|
|
#endif
|
|
|
|
char *substitutename = NULL;
|
2017-01-21 06:45:08 +08:00
|
|
|
size_t substitutename_len;
|
2017-07-27 05:10:07 +08:00
|
|
|
size_t substitutename_off = 0;
|
2017-01-21 09:11:24 +08:00
|
|
|
wchar_t tmpsubstname[MAXPATHLEN];
|
2009-06-16 08:13:56 +08:00
|
|
|
|
|
|
|
if(++(*ll) > LINK_MAX) {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
|
|
|
|
2017-12-07 23:52:23 +08:00
|
|
|
hLink = CreateFileW(pathw,
|
|
|
|
0,
|
|
|
|
PHP_WIN32_IOUTIL_DEFAULT_SHARE_MODE,
|
|
|
|
NULL,
|
|
|
|
OPEN_EXISTING,
|
|
|
|
FILE_FLAG_OPEN_REPARSE_POINT|FILE_FLAG_BACKUP_SEMANTICS,
|
|
|
|
NULL);
|
2009-06-16 08:13:56 +08:00
|
|
|
if(hLink == INVALID_HANDLE_VALUE) {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
|
|
|
|
2013-10-24 20:12:37 +08:00
|
|
|
pbuffer = (REPARSE_DATA_BUFFER *)do_alloca(MAXIMUM_REPARSE_DATA_BUFFER_SIZE, use_heap_large);
|
2011-01-10 08:43:08 +08:00
|
|
|
if (pbuffer == NULL) {
|
2016-05-11 01:59:21 +08:00
|
|
|
CloseHandle(hLink);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2011-01-10 08:43:08 +08:00
|
|
|
}
|
2009-06-16 08:13:56 +08:00
|
|
|
if(!DeviceIoControl(hLink, FSCTL_GET_REPARSE_POINT, NULL, 0, pbuffer, MAXIMUM_REPARSE_DATA_BUFFER_SIZE, &retlength, NULL)) {
|
2019-08-20 01:44:37 +08:00
|
|
|
BY_HANDLE_FILE_INFORMATION fileInformation;
|
|
|
|
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
2019-08-20 01:44:37 +08:00
|
|
|
if ((dataw.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) &&
|
|
|
|
(dataw.dwReserved0 & ~IO_REPARSE_TAG_CLOUD_MASK) == IO_REPARSE_TAG_CLOUD &&
|
|
|
|
EG(windows_version_info).dwMajorVersion >= 10 &&
|
|
|
|
EG(windows_version_info).dwMinorVersion == 0 &&
|
|
|
|
EG(windows_version_info).dwBuildNumber >= 18362 &&
|
|
|
|
GetFileInformationByHandle(hLink, &fileInformation) &&
|
|
|
|
!(fileInformation.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
|
|
|
|
dataw.dwFileAttributes = fileInformation.dwFileAttributes;
|
|
|
|
CloseHandle(hLink);
|
|
|
|
(*ll)--;
|
2020-07-27 23:50:41 +08:00
|
|
|
goto retry;
|
2019-08-20 01:44:37 +08:00
|
|
|
}
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
2009-06-16 08:13:56 +08:00
|
|
|
CloseHandle(hLink);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
CloseHandle(hLink);
|
|
|
|
|
|
|
|
if(pbuffer->ReparseTag == IO_REPARSE_TAG_SYMLINK) {
|
|
|
|
reparsetarget = pbuffer->SymbolicLinkReparseBuffer.ReparseTarget;
|
|
|
|
isabsolute = (pbuffer->SymbolicLinkReparseBuffer.Flags == 0) ? 1 : 0;
|
2017-01-21 09:17:06 +08:00
|
|
|
#if VIRTUAL_CWD_DEBUG
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
printname = php_win32_ioutil_w_to_any(reparsetarget + pbuffer->MountPointReparseBuffer.PrintNameOffset / sizeof(WCHAR));
|
|
|
|
if (!printname) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
}
|
2017-01-21 09:17:06 +08:00
|
|
|
#endif
|
2009-09-15 02:46:56 +08:00
|
|
|
|
|
|
|
substitutename_len = pbuffer->MountPointReparseBuffer.SubstituteNameLength / sizeof(WCHAR);
|
2017-01-24 20:24:11 +08:00
|
|
|
if (substitutename_len >= MAXPATHLEN) {
|
2017-01-21 09:11:24 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2017-01-21 09:11:24 +08:00
|
|
|
}
|
|
|
|
memmove(tmpsubstname, reparsetarget + pbuffer->MountPointReparseBuffer.SubstituteNameOffset / sizeof(WCHAR), pbuffer->MountPointReparseBuffer.SubstituteNameLength);
|
|
|
|
tmpsubstname[substitutename_len] = L'\0';
|
|
|
|
substitutename = php_win32_cp_conv_w_to_any(tmpsubstname, substitutename_len, &substitutename_len);
|
2017-01-24 20:24:11 +08:00
|
|
|
if (!substitutename || substitutename_len >= MAXPATHLEN) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
2017-01-24 20:24:11 +08:00
|
|
|
free(substitutename);
|
2017-01-21 09:17:06 +08:00
|
|
|
#if VIRTUAL_CWD_DEBUG
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free(printname);
|
2017-01-21 09:17:06 +08:00
|
|
|
#endif
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
}
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
|
|
|
else if(pbuffer->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT) {
|
|
|
|
isabsolute = 1;
|
2009-09-15 02:46:56 +08:00
|
|
|
reparsetarget = pbuffer->MountPointReparseBuffer.ReparseTarget;
|
2017-01-21 09:17:06 +08:00
|
|
|
#if VIRTUAL_CWD_DEBUG
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
printname = php_win32_ioutil_w_to_any(reparsetarget + pbuffer->MountPointReparseBuffer.PrintNameOffset / sizeof(WCHAR));
|
|
|
|
if (!printname) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
}
|
2017-01-21 09:17:06 +08:00
|
|
|
#endif
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
|
2009-09-15 02:46:56 +08:00
|
|
|
|
|
|
|
substitutename_len = pbuffer->MountPointReparseBuffer.SubstituteNameLength / sizeof(WCHAR);
|
2017-01-24 20:24:11 +08:00
|
|
|
if (substitutename_len >= MAXPATHLEN) {
|
2017-01-21 09:11:24 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2017-01-21 09:11:24 +08:00
|
|
|
}
|
|
|
|
memmove(tmpsubstname, reparsetarget + pbuffer->MountPointReparseBuffer.SubstituteNameOffset / sizeof(WCHAR), pbuffer->MountPointReparseBuffer.SubstituteNameLength);
|
|
|
|
tmpsubstname[substitutename_len] = L'\0';
|
|
|
|
substitutename = php_win32_cp_conv_w_to_any(tmpsubstname, substitutename_len, &substitutename_len);
|
2017-01-24 20:24:11 +08:00
|
|
|
if (!substitutename || substitutename_len >= MAXPATHLEN) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
2017-01-24 20:24:11 +08:00
|
|
|
free(substitutename);
|
2017-01-21 09:17:06 +08:00
|
|
|
#if VIRTUAL_CWD_DEBUG
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free(printname);
|
2017-01-21 09:17:06 +08:00
|
|
|
#endif
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
}
|
2012-11-01 05:41:03 +08:00
|
|
|
}
|
2017-11-08 18:52:42 +08:00
|
|
|
else if (pbuffer->ReparseTag == IO_REPARSE_TAG_DEDUP ||
|
|
|
|
/* Starting with 1709. */
|
2019-08-20 01:44:37 +08:00
|
|
|
(pbuffer->ReparseTag & ~IO_REPARSE_TAG_CLOUD_MASK) == IO_REPARSE_TAG_CLOUD ||
|
2020-04-24 23:24:25 +08:00
|
|
|
IO_REPARSE_TAG_ONEDRIVE == pbuffer->ReparseTag ||
|
|
|
|
IO_REPARSE_TAG_ACTIVISION_HSM == pbuffer->ReparseTag ||
|
|
|
|
IO_REPARSE_TAG_PROJFS == pbuffer->ReparseTag) {
|
2012-11-01 05:41:03 +08:00
|
|
|
isabsolute = 1;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
substitutename = malloc((len + 1) * sizeof(char));
|
|
|
|
if (!substitutename) {
|
|
|
|
free_alloca(pbuffer, use_heap_large);
|
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
}
|
2012-11-01 05:41:03 +08:00
|
|
|
memcpy(substitutename, path, len + 1);
|
|
|
|
substitutename_len = len;
|
2009-09-15 02:46:56 +08:00
|
|
|
} else {
|
2014-09-22 15:37:40 +08:00
|
|
|
/* XXX this might be not the end, restart handling with REPARSE_GUID_DATA_BUFFER should be implemented. */
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
|
|
|
|
2009-09-15 02:46:56 +08:00
|
|
|
if(isabsolute && substitutename_len > 4) {
|
2010-09-10 22:02:19 +08:00
|
|
|
/* Do not resolve volumes (for now). A mounted point can
|
2009-09-02 01:50:59 +08:00
|
|
|
target a volume without a drive, it is not certain that
|
2010-09-10 22:02:19 +08:00
|
|
|
all IO functions we use in php and its deps support
|
2009-09-02 01:50:59 +08:00
|
|
|
path with volume GUID instead of the DOS way, like:
|
|
|
|
d:\test\mnt\foo
|
|
|
|
\\?\Volume{62d1c3f8-83b9-11de-b108-806e6f6e6963}\foo
|
|
|
|
*/
|
2010-09-10 22:02:19 +08:00
|
|
|
if (strncmp(substitutename, "\\??\\Volume{",11) == 0
|
2010-08-26 22:23:17 +08:00
|
|
|
|| strncmp(substitutename, "\\\\?\\Volume{",11) == 0
|
|
|
|
|| strncmp(substitutename, "\\??\\UNC\\", 8) == 0
|
|
|
|
) {
|
2010-08-26 22:23:48 +08:00
|
|
|
isVolume = TRUE;
|
2009-09-15 02:46:56 +08:00
|
|
|
substitutename_off = 0;
|
2009-08-29 20:21:47 +08:00
|
|
|
} else
|
2009-09-02 01:50:59 +08:00
|
|
|
/* do not use the \??\ and \\?\ prefix*/
|
2010-09-10 22:02:19 +08:00
|
|
|
if (strncmp(substitutename, "\\??\\", 4) == 0
|
2009-09-15 02:46:56 +08:00
|
|
|
|| strncmp(substitutename, "\\\\?\\", 4) == 0) {
|
|
|
|
substitutename_off = 4;
|
2009-08-27 04:44:05 +08:00
|
|
|
}
|
|
|
|
}
|
2009-09-15 02:46:56 +08:00
|
|
|
|
2009-09-02 01:50:59 +08:00
|
|
|
if (!isVolume) {
|
2011-01-10 08:43:08 +08:00
|
|
|
char * tmp2 = substitutename + substitutename_off;
|
2017-07-27 05:10:07 +08:00
|
|
|
for (bufindex = 0; bufindex + substitutename_off < substitutename_len; bufindex++) {
|
2011-01-10 08:43:08 +08:00
|
|
|
*(path + bufindex) = *(tmp2 + bufindex);
|
2009-09-02 01:50:59 +08:00
|
|
|
}
|
2009-08-27 04:44:05 +08:00
|
|
|
|
2009-09-02 01:50:59 +08:00
|
|
|
*(path + bufindex) = 0;
|
|
|
|
j = bufindex;
|
|
|
|
} else {
|
2009-09-03 06:59:58 +08:00
|
|
|
j = len;
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
2009-09-15 02:46:56 +08:00
|
|
|
|
|
|
|
|
|
|
|
#if VIRTUAL_CWD_DEBUG
|
|
|
|
fprintf(stderr, "reparse: print: %s ", printname);
|
|
|
|
fprintf(stderr, "sub: %s ", substitutename);
|
|
|
|
fprintf(stderr, "resolved: %s ", path);
|
2017-01-21 09:17:06 +08:00
|
|
|
free(printname);
|
2009-09-15 02:46:56 +08:00
|
|
|
#endif
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(pbuffer, use_heap_large);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free(substitutename);
|
2009-06-16 08:13:56 +08:00
|
|
|
|
|
|
|
if(isabsolute == 1) {
|
2009-08-29 20:21:47 +08:00
|
|
|
if (!((j == 3) && (path[1] == ':') && (path[2] == '\\'))) {
|
|
|
|
/* use_realpath is 0 in the call below coz path is absolute*/
|
2014-12-14 06:06:14 +08:00
|
|
|
j = tsrm_realpath_r(path, 0, j, ll, t, 0, is_dir, &directory);
|
2017-07-27 05:10:07 +08:00
|
|
|
if(j == (size_t)-1) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2009-08-29 20:21:47 +08:00
|
|
|
}
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
if(i + j >= MAXPATHLEN - 1) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
memmove(path+i, path, j+1);
|
|
|
|
memcpy(path, tmp, i-1);
|
|
|
|
path[i-1] = DEFAULT_SLASH;
|
2014-12-14 06:06:14 +08:00
|
|
|
j = tsrm_realpath_r(path, start, i + j, ll, t, use_realpath, is_dir, &directory);
|
2017-07-27 05:10:07 +08:00
|
|
|
if(j == (size_t)-1) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
|
|
|
}
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
directory = (dataw.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY);
|
2009-06-16 08:13:56 +08:00
|
|
|
|
|
|
|
if(link_is_dir) {
|
|
|
|
*link_is_dir = directory;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
if (save) {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
directory = (dataw.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
|
2009-06-16 08:13:56 +08:00
|
|
|
if (is_dir && !directory) {
|
|
|
|
/* not a directory */
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
|
|
|
}
|
2006-09-20 15:35:59 +08:00
|
|
|
#else
|
2010-09-13 18:27:55 +08:00
|
|
|
if (save && php_sys_lstat(path, &st) < 0) {
|
2008-08-12 16:01:24 +08:00
|
|
|
if (use_realpath == CWD_REALPATH) {
|
|
|
|
/* file not found */
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
|
|
|
/* continue resolution anyway but don't save result in the cache */
|
|
|
|
save = 0;
|
2007-07-24 17:07:33 +08:00
|
|
|
}
|
|
|
|
|
2013-10-24 20:12:37 +08:00
|
|
|
tmp = do_alloca(len+1, use_heap);
|
2008-08-12 16:01:24 +08:00
|
|
|
memcpy(tmp, path, len+1);
|
2006-09-20 15:35:59 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
if (save && S_ISLNK(st.st_mode)) {
|
2017-07-27 05:10:07 +08:00
|
|
|
if (++(*ll) > LINK_MAX || (j = (size_t)php_sys_readlink(tmp, path, MAXPATHLEN)) == (size_t)-1) {
|
2010-08-27 18:09:52 +08:00
|
|
|
/* too many links or broken symlinks */
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
|
|
|
path[j] = 0;
|
|
|
|
if (IS_ABSOLUTE_PATH(path, j)) {
|
2014-12-14 06:06:14 +08:00
|
|
|
j = tsrm_realpath_r(path, 1, j, ll, t, use_realpath, is_dir, &directory);
|
2017-07-27 05:10:07 +08:00
|
|
|
if (j == (size_t)-1) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
|
|
|
} else {
|
2010-08-27 18:09:52 +08:00
|
|
|
if (i + j >= MAXPATHLEN-1) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1; /* buffer overflow */
|
2010-08-27 18:09:52 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
memmove(path+i, path, j+1);
|
|
|
|
memcpy(path, tmp, i-1);
|
|
|
|
path[i-1] = DEFAULT_SLASH;
|
2014-12-14 06:06:14 +08:00
|
|
|
j = tsrm_realpath_r(path, start, i + j, ll, t, use_realpath, is_dir, &directory);
|
2017-07-27 05:10:07 +08:00
|
|
|
if (j == (size_t)-1) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
|
|
|
}
|
2008-08-18 18:28:17 +08:00
|
|
|
if (link_is_dir) {
|
|
|
|
*link_is_dir = directory;
|
|
|
|
}
|
2006-09-20 15:35:59 +08:00
|
|
|
} else {
|
2008-08-12 16:01:24 +08:00
|
|
|
if (save) {
|
|
|
|
directory = S_ISDIR(st.st_mode);
|
2008-08-15 21:31:58 +08:00
|
|
|
if (link_is_dir) {
|
|
|
|
*link_is_dir = directory;
|
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
if (is_dir && !directory) {
|
|
|
|
/* not a directory */
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
#endif
|
2017-07-27 05:10:07 +08:00
|
|
|
if (i <= start + 1) {
|
2008-08-12 16:01:24 +08:00
|
|
|
j = start;
|
|
|
|
} else {
|
|
|
|
/* some leading directories may be unaccessable */
|
2014-12-14 06:06:14 +08:00
|
|
|
j = tsrm_realpath_r(path, start, i-1, ll, t, save ? CWD_FILEPATH : use_realpath, 1, NULL);
|
2017-07-28 05:38:04 +08:00
|
|
|
if (j > start && j != (size_t)-1) {
|
2008-08-12 16:01:24 +08:00
|
|
|
path[j++] = DEFAULT_SLASH;
|
|
|
|
}
|
|
|
|
}
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2017-07-27 05:10:07 +08:00
|
|
|
if (j == (size_t)-1 || j + len >= MAXPATHLEN - 1 + i) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
|
|
|
if (save) {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
size_t sz;
|
|
|
|
char *tmp_path = php_win32_ioutil_conv_w_to_any(dataw.cFileName, PHP_WIN32_CP_IGNORE_LEN, &sz);
|
|
|
|
if (!tmp_path) {
|
|
|
|
free_alloca(tmp, use_heap);
|
|
|
|
FREE_PATHW()
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
}
|
2017-07-27 05:10:07 +08:00
|
|
|
i = sz;
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
memcpy(path+j, tmp_path, i+1);
|
|
|
|
free(tmp_path);
|
2008-08-19 23:00:20 +08:00
|
|
|
j += i;
|
2008-08-12 16:01:24 +08:00
|
|
|
} else {
|
|
|
|
/* use the original file or directory name as it wasn't found */
|
|
|
|
memcpy(path+j, tmp+i, len-i+1);
|
2008-08-19 23:00:20 +08:00
|
|
|
j += (len-i);
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
2009-06-16 08:13:56 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
#else
|
2017-07-27 05:10:07 +08:00
|
|
|
if (j == (size_t)-1 || j + len >= MAXPATHLEN - 1 + i) {
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
2017-07-27 05:10:07 +08:00
|
|
|
return (size_t)-1;
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
|
|
|
memcpy(path+j, tmp+i, len-i+1);
|
|
|
|
j += (len-i);
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
2006-09-20 15:35:59 +08:00
|
|
|
#endif
|
2007-04-04 04:20:56 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
if (save && start && CWDG(realpath_cache_size_limit)) {
|
|
|
|
/* save absolute path in the cache */
|
2014-12-14 06:06:14 +08:00
|
|
|
realpath_cache_add(tmp, len, path, j, directory, *t);
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(tmp, use_heap);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#ifdef ZEND_WIN32
|
|
|
|
FREE_PATHW()
|
|
|
|
#undef FREE_PATHW
|
|
|
|
#endif
|
2008-08-12 16:01:24 +08:00
|
|
|
return j;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
/* Resolve path relatively to state and put the real path into state */
|
|
|
|
/* returns 0 for ok, 1 for error */
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_file_ex(cwd_state *state, const char *path, verify_path_func verify_path, int use_realpath) /* {{{ */
|
2008-08-12 16:01:24 +08:00
|
|
|
{
|
2017-07-27 05:10:07 +08:00
|
|
|
size_t path_length = strlen(path);
|
2017-12-15 20:23:34 +08:00
|
|
|
char resolved_path[MAXPATHLEN] = {0};
|
2017-07-27 05:10:07 +08:00
|
|
|
size_t start = 1;
|
2008-08-12 16:01:24 +08:00
|
|
|
int ll = 0;
|
|
|
|
time_t t;
|
|
|
|
int ret;
|
|
|
|
int add_slash;
|
2009-10-20 07:41:14 +08:00
|
|
|
void *tmp;
|
2003-03-17 21:40:45 +08:00
|
|
|
|
2017-07-27 05:10:07 +08:00
|
|
|
if (!path_length || path_length >= MAXPATHLEN-1) {
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2010-02-01 23:22:31 +08:00
|
|
|
_set_errno(EINVAL);
|
|
|
|
#else
|
2010-02-01 23:31:58 +08:00
|
|
|
errno = EINVAL;
|
2010-02-01 23:22:31 +08:00
|
|
|
#endif
|
2008-08-12 16:01:24 +08:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
#if VIRTUAL_CWD_DEBUG
|
|
|
|
fprintf(stderr,"cwd = %s path = %s\n", state->cwd, path);
|
2001-05-03 23:50:37 +08:00
|
|
|
#endif
|
2008-08-12 16:01:24 +08:00
|
|
|
|
|
|
|
/* cwd_length can be 0 when getcwd() fails.
|
|
|
|
* This can happen under solaris when a dir does not have read permissions
|
|
|
|
* but *does* have execute permissions */
|
|
|
|
if (!IS_ABSOLUTE_PATH(path, path_length)) {
|
|
|
|
if (state->cwd_length == 0) {
|
|
|
|
/* resolve relative path */
|
|
|
|
start = 0;
|
|
|
|
memcpy(resolved_path , path, path_length + 1);
|
|
|
|
} else {
|
2017-07-27 05:10:07 +08:00
|
|
|
size_t state_cwd_length = state->cwd_length;
|
2006-09-20 15:35:59 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2008-08-12 16:01:24 +08:00
|
|
|
if (IS_SLASH(path[0])) {
|
2009-01-22 03:07:40 +08:00
|
|
|
if (state->cwd[1] == ':') {
|
|
|
|
/* Copy only the drive name */
|
|
|
|
state_cwd_length = 2;
|
|
|
|
} else if (IS_UNC_PATH(state->cwd, state->cwd_length)) {
|
|
|
|
/* Copy only the share name */
|
|
|
|
state_cwd_length = 2;
|
|
|
|
while (IS_SLASH(state->cwd[state_cwd_length])) {
|
|
|
|
state_cwd_length++;
|
2010-09-10 22:02:19 +08:00
|
|
|
}
|
2009-01-22 03:07:40 +08:00
|
|
|
while (state->cwd[state_cwd_length] &&
|
2010-08-27 18:09:52 +08:00
|
|
|
!IS_SLASH(state->cwd[state_cwd_length])) {
|
2009-01-22 03:07:40 +08:00
|
|
|
state_cwd_length++;
|
2010-09-10 22:02:19 +08:00
|
|
|
}
|
2009-01-22 03:07:40 +08:00
|
|
|
while (IS_SLASH(state->cwd[state_cwd_length])) {
|
|
|
|
state_cwd_length++;
|
2010-09-10 22:02:19 +08:00
|
|
|
}
|
2009-01-22 03:07:40 +08:00
|
|
|
while (state->cwd[state_cwd_length] &&
|
2010-08-27 18:09:52 +08:00
|
|
|
!IS_SLASH(state->cwd[state_cwd_length])) {
|
2009-01-22 03:07:40 +08:00
|
|
|
state_cwd_length++;
|
2010-09-10 22:02:19 +08:00
|
|
|
}
|
2009-01-22 03:07:40 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
}
|
2006-09-20 15:35:59 +08:00
|
|
|
#endif
|
2008-08-12 16:01:24 +08:00
|
|
|
if (path_length + state_cwd_length + 1 >= MAXPATHLEN-1) {
|
|
|
|
return 1;
|
2003-03-17 21:40:45 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
memcpy(resolved_path, state->cwd, state_cwd_length);
|
2012-01-30 18:08:11 +08:00
|
|
|
if (resolved_path[state_cwd_length-1] == DEFAULT_SLASH) {
|
|
|
|
memcpy(resolved_path + state_cwd_length, path, path_length + 1);
|
|
|
|
path_length += state_cwd_length;
|
|
|
|
} else {
|
|
|
|
resolved_path[state_cwd_length] = DEFAULT_SLASH;
|
|
|
|
memcpy(resolved_path + state_cwd_length + 1, path, path_length + 1);
|
|
|
|
path_length += state_cwd_length + 1;
|
|
|
|
}
|
2001-05-06 00:05:19 +08:00
|
|
|
}
|
2010-09-10 22:02:19 +08:00
|
|
|
} else {
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2009-01-22 03:07:40 +08:00
|
|
|
if (path_length > 2 && path[1] == ':' && !IS_SLASH(path[2])) {
|
|
|
|
resolved_path[0] = path[0];
|
|
|
|
resolved_path[1] = ':';
|
|
|
|
resolved_path[2] = DEFAULT_SLASH;
|
|
|
|
memcpy(resolved_path + 3, path + 2, path_length - 1);
|
|
|
|
path_length++;
|
|
|
|
} else
|
|
|
|
#endif
|
|
|
|
memcpy(resolved_path, path, path_length + 1);
|
2010-09-10 22:02:19 +08:00
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2008-08-12 16:01:24 +08:00
|
|
|
if (memchr(resolved_path, '*', path_length) ||
|
2010-08-27 18:09:52 +08:00
|
|
|
memchr(resolved_path, '?', path_length)) {
|
2008-08-12 16:01:24 +08:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
#endif
|
2006-09-20 15:35:59 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2008-08-12 16:01:24 +08:00
|
|
|
if (IS_UNC_PATH(resolved_path, path_length)) {
|
|
|
|
/* skip UNC name */
|
|
|
|
resolved_path[0] = DEFAULT_SLASH;
|
|
|
|
resolved_path[1] = DEFAULT_SLASH;
|
|
|
|
start = 2;
|
|
|
|
while (!IS_SLASH(resolved_path[start])) {
|
|
|
|
if (resolved_path[start] == 0) {
|
|
|
|
goto verify;
|
2007-11-13 17:47:06 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
resolved_path[start] = toupper(resolved_path[start]);
|
|
|
|
start++;
|
|
|
|
}
|
|
|
|
resolved_path[start++] = DEFAULT_SLASH;
|
|
|
|
while (!IS_SLASH(resolved_path[start])) {
|
|
|
|
if (resolved_path[start] == 0) {
|
|
|
|
goto verify;
|
2007-11-13 17:47:06 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
resolved_path[start] = toupper(resolved_path[start]);
|
|
|
|
start++;
|
2007-02-19 22:58:07 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
resolved_path[start++] = DEFAULT_SLASH;
|
|
|
|
} else if (IS_ABSOLUTE_PATH(resolved_path, path_length)) {
|
|
|
|
/* skip DRIVE name */
|
|
|
|
resolved_path[0] = toupper(resolved_path[0]);
|
|
|
|
resolved_path[2] = DEFAULT_SLASH;
|
|
|
|
start = 3;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2008-08-19 23:00:20 +08:00
|
|
|
add_slash = (use_realpath != CWD_REALPATH) && path_length > 0 && IS_SLASH(resolved_path[path_length-1]);
|
2008-08-12 16:01:24 +08:00
|
|
|
t = CWDG(realpath_cache_ttl) ? 0 : -1;
|
2014-12-14 06:06:14 +08:00
|
|
|
path_length = tsrm_realpath_r(resolved_path, start, path_length, &ll, &t, use_realpath, 0, NULL);
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2017-07-27 05:10:07 +08:00
|
|
|
if (path_length == (size_t)-1) {
|
2009-01-22 03:07:40 +08:00
|
|
|
errno = ENOENT;
|
2008-08-12 16:01:24 +08:00
|
|
|
return 1;
|
|
|
|
}
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
if (!start && !path_length) {
|
|
|
|
resolved_path[path_length++] = '.';
|
|
|
|
}
|
2017-12-15 20:23:34 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
if (add_slash && path_length && !IS_SLASH(resolved_path[path_length-1])) {
|
|
|
|
if (path_length >= MAXPATHLEN-1) {
|
|
|
|
return -1;
|
2003-03-17 21:40:45 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
resolved_path[path_length++] = DEFAULT_SLASH;
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
2008-08-12 16:01:24 +08:00
|
|
|
resolved_path[path_length] = 0;
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2008-08-12 16:01:24 +08:00
|
|
|
verify:
|
2007-10-23 13:57:25 +08:00
|
|
|
#endif
|
2008-08-12 16:01:24 +08:00
|
|
|
if (verify_path) {
|
|
|
|
cwd_state old_state;
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
CWD_STATE_COPY(&old_state, state);
|
|
|
|
state->cwd_length = path_length;
|
2009-10-20 07:41:14 +08:00
|
|
|
|
2013-10-17 16:40:43 +08:00
|
|
|
tmp = erealloc(state->cwd, state->cwd_length+1);
|
2009-10-20 07:41:14 +08:00
|
|
|
state->cwd = (char *) tmp;
|
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
memcpy(state->cwd, resolved_path, state->cwd_length+1);
|
|
|
|
if (verify_path(state)) {
|
|
|
|
CWD_STATE_FREE(state);
|
|
|
|
*state = old_state;
|
|
|
|
ret = 1;
|
|
|
|
} else {
|
|
|
|
CWD_STATE_FREE(&old_state);
|
|
|
|
ret = 0;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
} else {
|
2008-08-12 16:01:24 +08:00
|
|
|
state->cwd_length = path_length;
|
2013-10-17 16:40:43 +08:00
|
|
|
tmp = erealloc(state->cwd, state->cwd_length+1);
|
2009-10-20 07:41:14 +08:00
|
|
|
state->cwd = (char *) tmp;
|
|
|
|
|
2008-08-12 16:01:24 +08:00
|
|
|
memcpy(state->cwd, resolved_path, state->cwd_length+1);
|
2000-09-04 02:18:13 +08:00
|
|
|
ret = 0;
|
|
|
|
}
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2000-09-04 02:18:13 +08:00
|
|
|
#if VIRTUAL_CWD_DEBUG
|
|
|
|
fprintf (stderr, "virtual_file_ex() = %s\n",state->cwd);
|
|
|
|
#endif
|
|
|
|
return (ret);
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_chdir(const char *path) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2014-12-14 06:06:14 +08:00
|
|
|
return virtual_file_ex(&CWDG(cwd), path, php_is_dir_ok, CWD_REALPATH)?-1:0;
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_chdir_file(const char *path, int (*p_chdir)(const char *path)) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2018-03-19 18:44:27 +08:00
|
|
|
size_t length = strlen(path);
|
2000-09-04 02:18:13 +08:00
|
|
|
char *temp;
|
|
|
|
int retval;
|
2013-10-24 20:12:37 +08:00
|
|
|
ALLOCA_FLAG(use_heap)
|
2000-09-04 02:18:13 +08:00
|
|
|
|
|
|
|
if (length == 0) {
|
|
|
|
return 1; /* Can't cd to empty string */
|
2010-09-10 22:02:19 +08:00
|
|
|
}
|
2018-03-19 18:44:27 +08:00
|
|
|
while(--length < SIZE_MAX && !IS_SLASH(path[length])) {
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
|
|
|
|
2018-03-19 18:44:27 +08:00
|
|
|
if (length == SIZE_MAX) {
|
2000-10-03 23:08:37 +08:00
|
|
|
/* No directory only file name */
|
|
|
|
errno = ENOENT;
|
|
|
|
return -1;
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
|
|
|
|
2002-05-29 16:41:21 +08:00
|
|
|
if (length == COPY_WHEN_ABSOLUTE(path) && IS_ABSOLUTE_PATH(path, length+1)) { /* Also use trailing slash if this is absolute */
|
2000-09-04 02:18:13 +08:00
|
|
|
length++;
|
|
|
|
}
|
2013-10-24 20:12:37 +08:00
|
|
|
temp = (char *) do_alloca(length+1, use_heap);
|
2000-09-04 02:18:13 +08:00
|
|
|
memcpy(temp, path, length);
|
|
|
|
temp[length] = 0;
|
|
|
|
#if VIRTUAL_CWD_DEBUG
|
|
|
|
fprintf (stderr, "Changing directory to %s\n", temp);
|
|
|
|
#endif
|
2014-12-14 06:06:14 +08:00
|
|
|
retval = p_chdir(temp);
|
2013-10-24 20:12:37 +08:00
|
|
|
free_alloca(temp, use_heap);
|
2000-09-04 02:18:13 +08:00
|
|
|
return retval;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API char *virtual_realpath(const char *path, char *real_path) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
2003-06-25 13:05:06 +08:00
|
|
|
char *retval;
|
2007-07-10 21:21:11 +08:00
|
|
|
char cwd[MAXPATHLEN];
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2007-07-10 21:21:11 +08:00
|
|
|
/* realpath("") returns CWD */
|
|
|
|
if (!*path) {
|
2013-10-17 16:40:43 +08:00
|
|
|
new_state.cwd = (char*)emalloc(1);
|
2007-07-10 21:21:11 +08:00
|
|
|
new_state.cwd[0] = '\0';
|
2010-09-10 22:02:19 +08:00
|
|
|
new_state.cwd_length = 0;
|
2010-08-27 18:09:52 +08:00
|
|
|
if (VCWD_GETCWD(cwd, MAXPATHLEN)) {
|
|
|
|
path = cwd;
|
2007-07-10 21:21:11 +08:00
|
|
|
}
|
2014-09-19 21:59:55 +08:00
|
|
|
} else if (!IS_ABSOLUTE_PATH(path, strlen(path))) {
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2007-07-10 21:21:11 +08:00
|
|
|
} else {
|
2014-09-19 21:59:55 +08:00
|
|
|
new_state.cwd = (char*)emalloc(1);
|
|
|
|
new_state.cwd[0] = '\0';
|
|
|
|
new_state.cwd_length = 0;
|
2007-07-10 21:21:11 +08:00
|
|
|
}
|
2009-06-26 15:39:42 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, path, NULL, CWD_REALPATH)==0) {
|
2017-07-27 05:10:07 +08:00
|
|
|
size_t len = new_state.cwd_length>MAXPATHLEN-1?MAXPATHLEN-1:new_state.cwd_length;
|
2003-06-25 13:05:06 +08:00
|
|
|
|
2000-09-04 02:18:13 +08:00
|
|
|
memcpy(real_path, new_state.cwd, len);
|
|
|
|
real_path[len] = '\0';
|
2003-06-25 13:05:06 +08:00
|
|
|
retval = real_path;
|
|
|
|
} else {
|
|
|
|
retval = NULL;
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
|
|
|
|
2003-06-25 13:05:06 +08:00
|
|
|
CWD_STATE_FREE(&new_state);
|
|
|
|
return retval;
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_filepath_ex(const char *path, char **filepath, verify_path_func verify_path) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int retval;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
retval = virtual_file_ex(&new_state, path, verify_path, CWD_FILEPATH);
|
2000-09-04 02:18:13 +08:00
|
|
|
|
|
|
|
*filepath = new_state.cwd;
|
|
|
|
|
|
|
|
return retval;
|
2000-11-22 12:59:32 +08:00
|
|
|
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-11-22 12:59:32 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_filepath(const char *path, char **filepath) /* {{{ */
|
2000-11-22 12:59:32 +08:00
|
|
|
{
|
2014-12-14 06:06:14 +08:00
|
|
|
return virtual_filepath_ex(path, filepath, php_is_file_ok);
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API FILE *virtual_fopen(const char *path, const char *mode) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
FILE *f;
|
|
|
|
|
|
|
|
if (path[0] == '\0') { /* Fail to open empty path */
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, path, NULL, CWD_EXPAND)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#ifdef ZEND_WIN32
|
|
|
|
f = php_win32_ioutil_fopen(new_state.cwd, mode);
|
|
|
|
#else
|
2000-09-04 02:18:13 +08:00
|
|
|
f = fopen(new_state.cwd, mode);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#endif
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
|
|
|
|
2000-09-04 02:18:13 +08:00
|
|
|
return f;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_access(const char *pathname, int mode) /* {{{ */
|
2002-10-04 09:04:00 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int ret;
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2002-10-04 09:04:00 +08:00
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, pathname, NULL, CWD_REALPATH)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-08-05 21:17:50 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2002-10-04 09:04:00 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#if defined(ZEND_WIN32)
|
2014-12-14 06:06:14 +08:00
|
|
|
ret = tsrm_win32_access(new_state.cwd, mode);
|
2003-06-25 13:05:06 +08:00
|
|
|
#else
|
2002-10-04 09:04:00 +08:00
|
|
|
ret = access(new_state.cwd, mode);
|
2003-06-25 13:05:06 +08:00
|
|
|
#endif
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2002-10-04 09:04:00 +08:00
|
|
|
return ret;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2002-10-04 09:04:00 +08:00
|
|
|
|
2000-09-04 02:58:46 +08:00
|
|
|
#if HAVE_UTIME
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_utime(const char *filename, struct utimbuf *buf) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, filename, NULL, CWD_REALPATH)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2008-08-15 23:00:11 +08:00
|
|
|
ret = win32_utime(new_state.cwd, buf);
|
|
|
|
#else
|
2000-09-04 02:18:13 +08:00
|
|
|
ret = utime(new_state.cwd, buf);
|
2008-08-15 23:00:11 +08:00
|
|
|
#endif
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return ret;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
#endif
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_chmod(const char *filename, mode_t mode) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, filename, NULL, CWD_REALPATH)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-11-05 18:58:44 +08:00
|
|
|
#ifdef ZEND_WIN32
|
|
|
|
{
|
|
|
|
mode_t _tmp = mode;
|
|
|
|
|
|
|
|
mode = 0;
|
|
|
|
|
|
|
|
if (_tmp & _S_IREAD) {
|
|
|
|
mode |= _S_IREAD;
|
|
|
|
}
|
|
|
|
if (_tmp & _S_IWRITE) {
|
|
|
|
mode |= _S_IWRITE;
|
|
|
|
}
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
ret = php_win32_ioutil_chmod(new_state.cwd, mode);
|
2015-11-05 18:58:44 +08:00
|
|
|
}
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#else
|
2000-09-04 02:18:13 +08:00
|
|
|
ret = chmod(new_state.cwd, mode);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#endif
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return ret;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2016-11-12 18:20:01 +08:00
|
|
|
#if !defined(ZEND_WIN32)
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_chown(const char *filename, uid_t owner, gid_t group, int link) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, filename, NULL, CWD_REALPATH)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2006-03-06 02:57:54 +08:00
|
|
|
if (link) {
|
|
|
|
#if HAVE_LCHOWN
|
|
|
|
ret = lchown(new_state.cwd, owner, group);
|
|
|
|
#else
|
|
|
|
ret = -1;
|
|
|
|
#endif
|
|
|
|
} else {
|
|
|
|
ret = chown(new_state.cwd, owner, group);
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return ret;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
#endif
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_open(const char *path, int flags, ...) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int f;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, path, NULL, CWD_FILEPATH)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
|
|
|
if (flags & O_CREAT) {
|
|
|
|
mode_t mode;
|
|
|
|
va_list arg;
|
|
|
|
|
|
|
|
va_start(arg, flags);
|
2001-11-18 05:15:38 +08:00
|
|
|
mode = (mode_t) va_arg(arg, int);
|
2000-09-04 02:18:13 +08:00
|
|
|
va_end(arg);
|
|
|
|
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#ifdef ZEND_WIN32
|
|
|
|
f = php_win32_ioutil_open(new_state.cwd, flags, mode);
|
|
|
|
#else
|
2000-09-04 02:18:13 +08:00
|
|
|
f = open(new_state.cwd, flags, mode);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#endif
|
2000-09-04 02:18:13 +08:00
|
|
|
} else {
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#ifdef ZEND_WIN32
|
|
|
|
f = php_win32_ioutil_open(new_state.cwd, flags);
|
|
|
|
#else
|
2000-09-04 02:18:13 +08:00
|
|
|
f = open(new_state.cwd, flags);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#endif
|
2010-09-10 22:02:19 +08:00
|
|
|
}
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return f;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_creat(const char *path, mode_t mode) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int f;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, path, NULL, CWD_FILEPATH)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
|
|
|
f = creat(new_state.cwd, mode);
|
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return f;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_rename(const char *oldname, const char *newname) /* {{{ */
|
2000-10-26 13:21:52 +08:00
|
|
|
{
|
|
|
|
cwd_state old_state;
|
|
|
|
cwd_state new_state;
|
|
|
|
int retval;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&old_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&old_state, oldname, NULL, CWD_EXPAND)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&old_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-10-26 13:21:52 +08:00
|
|
|
oldname = old_state.cwd;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, newname, NULL, CWD_EXPAND)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&old_state);
|
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-10-26 13:21:52 +08:00
|
|
|
newname = new_state.cwd;
|
2008-04-24 15:45:01 +08:00
|
|
|
|
|
|
|
/* rename on windows will fail if newname already exists.
|
|
|
|
MoveFileEx has to be used */
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2010-01-28 00:44:48 +08:00
|
|
|
/* MoveFileEx returns 0 on failure, other way 'round for this function */
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
retval = php_win32_ioutil_rename(oldname, newname);
|
2008-04-24 15:45:01 +08:00
|
|
|
#else
|
2000-10-26 13:21:52 +08:00
|
|
|
retval = rename(oldname, newname);
|
2008-04-24 15:45:01 +08:00
|
|
|
#endif
|
2000-10-26 13:21:52 +08:00
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&old_state);
|
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2013-10-18 20:01:16 +08:00
|
|
|
|
2000-10-26 13:21:52 +08:00
|
|
|
return retval;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_stat(const char *path, zend_stat_t *buf) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int retval;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, path, NULL, CWD_REALPATH)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2005-12-20 22:24:14 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2006-11-10 23:04:07 +08:00
|
|
|
retval = php_sys_stat(new_state.cwd, buf);
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return retval;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_lstat(const char *path, zend_stat_t *buf) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2002-11-05 07:24:15 +08:00
|
|
|
cwd_state new_state;
|
2000-09-04 02:18:13 +08:00
|
|
|
int retval;
|
|
|
|
|
2002-11-05 07:24:15 +08:00
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, path, NULL, CWD_EXPAND)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2002-11-05 07:24:15 +08:00
|
|
|
|
2010-09-01 17:49:53 +08:00
|
|
|
retval = php_sys_lstat(new_state.cwd, buf);
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return retval;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_unlink(const char *path) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2002-11-05 07:24:15 +08:00
|
|
|
cwd_state new_state;
|
2000-09-04 02:18:13 +08:00
|
|
|
int retval;
|
|
|
|
|
2002-11-05 07:24:15 +08:00
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, path, NULL, CWD_EXPAND)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#ifdef ZEND_WIN32
|
|
|
|
retval = php_win32_ioutil_unlink(new_state.cwd);
|
|
|
|
#else
|
2002-11-05 07:24:15 +08:00
|
|
|
retval = unlink(new_state.cwd);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#endif
|
2002-11-05 07:24:15 +08:00
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return retval;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_mkdir(const char *pathname, mode_t mode) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int retval;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, pathname, NULL, CWD_FILEPATH)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
retval = php_win32_ioutil_mkdir(new_state.cwd, mode);
|
2000-09-04 02:47:35 +08:00
|
|
|
#else
|
2000-09-04 02:18:13 +08:00
|
|
|
retval = mkdir(new_state.cwd, mode);
|
2000-09-04 02:47:35 +08:00
|
|
|
#endif
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return retval;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API int virtual_rmdir(const char *pathname) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
int retval;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, pathname, NULL, CWD_EXPAND)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return -1;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#ifdef ZEND_WIN32
|
|
|
|
retval = php_win32_ioutil_rmdir(new_state.cwd);
|
|
|
|
#else
|
2000-09-04 02:18:13 +08:00
|
|
|
retval = rmdir(new_state.cwd);
|
Fixed the UTF-8 and long path support in the streams on Windows.
Since long the default PHP charset is UTF-8, however the Windows part is
out of step with this important point. The current implementation in PHP
doesn't technically permit to handle UTF-8 filepath and several other
things. Till now, only the ANSI compatible APIs are being used. Here is more
about it
https://msdn.microsoft.com/en-us/library/windows/desktop/dd317752%28v=vs.85%29.aspx
The patch fixes not only issues with multibyte filenames under
incompatible codepages, but indirectly also issues with some other multibyte
encodings like BIG5, Shift-JIS, etc. by providing a clean way to access
filenames in UTF-8. Below is a small list of issues from the bug tracker,
that are getting fixed:
https://bugs.php.net/63401
https://bugs.php.net/41199
https://bugs.php.net/50203
https://bugs.php.net/71509
https://bugs.php.net/64699
https://bugs.php.net/64506
https://bugs.php.net/30195
https://bugs.php.net/65358
https://bugs.php.net/61315
https://bugs.php.net/70943
https://bugs.php.net/70903
https://bugs.php.net/63593
https://bugs.php.net/54977
https://bugs.php.net/54028
https://bugs.php.net/43148
https://bugs.php.net/30730
https://bugs.php.net/33350
https://bugs.php.net/35300
https://bugs.php.net/46990
https://bugs.php.net/61309
https://bugs.php.net/69333
https://bugs.php.net/45517
https://bugs.php.net/70551
https://bugs.php.net/50197
https://bugs.php.net/72200
https://bugs.php.net/37672
Yet more related tickets can for sure be found - on bugs.php.net, Stackoverflow
and Github. Some of the bugs are pretty recent, some descend to early
2000th, but the user comments in there last even till today. Just for example,
bug #30195 was opened in 2004, the latest comment in there was made in 2014. It
is certain, that these bugs descend not only to pure PHP use cases, but get also
redirected from the popular PHP based projects. Given the modern systems (and
those supported by PHP) are always based on NTFS, there is no excuse to keep
these issues unresolved.
The internalization approach on Windows is in many ways different from
UNIX and Linux, while it supports and is based on Unicode. It depends on the
current system code page, APIs used and exact kind how the binary was compiled
The locale doesn't affect the way Unicode or ANSI API work. PHP in particular
is being compiled without _UNICODE defined and this is conditioned by the
way we handle strings. Here is more about it
https://msdn.microsoft.com/en-us/library/tsbaswba.aspx
However, with any system code page ANSI functions automatically convert
paths to UTF-16. Paths in some encodings incompatible with the
current system code page, won't work correctly with ANSI APIs. PHP
till now only uses the ANSI Windows APIs.
For example, on a system with the current code page 1252, the paths
in cp1252 are supported and transparently converted to UTF-16 by the
ANSI functions. Once one wants to handle a filepath encoded with cp932 on
that particular system, an ANSI or a POSIX compatible function used in
PHP will produce an erroneous result. When trying to convert that cp932 path
to UTF-8 and passing to the ANSI functions, an ANSI function would
likely interpret the UTF-8 string as some string in the current code page and
create a filepath that represents every single byte of the UTF-8 string.
These behaviors are not only broken but also disregard the documented
INI settings.
This patch solves the issies with the multibyte paths on Windows by
intelligently enforcing the usage of the Unicode aware APIs. For
functions expect Unicode (fe CreateFileW, FindFirstFileW, etc.), arguments
will be converted to UTF-16 wide chars. For functions returning Unicode
aware data (fe GetCurrentDirectoryW, etc.), resulting wide string is
converted back to char's depending on the current PHP charset settings,
either to the current ANSI codepage (this is the behavior prior to this patch)
or to UTF-8 (the default behavior).
In a particular case, users might have to explicitly set
internal_encoding or default_charset, if filenames in ANSI codepage are
necessary. Current tests show no regressions and witness that this will be an
exotic case, the current default UTF-8 encoding is compatible with any
supported system. The dependency libraries are long switching to Unicode APIs,
so some tests were also added for extensions not directly related to streams.
At large, the patch brings over 150 related tests into the core. Those target
and was run on various environments with European, Asian, etc. codepages.
General PHP frameworks was tested and showed no regressions.
The impact on the current C code base is low, the most places affected
are the Windows only places in the three files tsrm_win32.c, zend_virtual_cwd.c
and plain_wrapper.c. The actual implementation of the most of the wide
char supporting functionality is in win32/ioutil.* and win32/codepage.*,
several low level functionsare extended in place to avoid reimplementation for
now. No performance impact was sighted. As previously mentioned, the ANSI APIs
used prior the patch perform Unicode conversions internally. Using the
Unicode APIs directly while doing custom conversions just retains the status
quo. The ways to optimize it are open (fe. by implementing caching for the
strings converted to wide variants).
The long path implementation is user transparent. If a path exceeds the
length of _MAX_PATH, it'll be automatically prefixed with \\?\. The MAXPATHLEN
is set to 2048 bytes.
Appreciation to Pierre Joye, Matt Ficken, @algo13 and others for tips, ideas
and testing.
Thanks.
2016-06-20 15:32:19 +08:00
|
|
|
#endif
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return retval;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2000-09-04 02:47:35 +08:00
|
|
|
DIR *opendir(const char *name);
|
|
|
|
#endif
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API DIR *virtual_opendir(const char *pathname) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
|
|
|
cwd_state new_state;
|
|
|
|
DIR *retval;
|
|
|
|
|
|
|
|
CWD_STATE_COPY(&new_state, &CWDG(cwd));
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, pathname, NULL, CWD_REALPATH)) {
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2006-09-28 23:48:35 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
2000-09-04 02:18:13 +08:00
|
|
|
|
|
|
|
retval = opendir(new_state.cwd);
|
|
|
|
|
2013-10-31 03:46:08 +08:00
|
|
|
CWD_STATE_FREE_ERR(&new_state);
|
2000-09-04 02:18:13 +08:00
|
|
|
return retval;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
2015-07-05 00:55:22 +08:00
|
|
|
#ifdef ZEND_WIN32
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API FILE *virtual_popen(const char *command, const char *type) /* {{{ */
|
2000-09-04 02:18:13 +08:00
|
|
|
{
|
2014-12-14 06:06:14 +08:00
|
|
|
return popen_ex(command, type, CWDG(cwd).cwd, NULL);
|
2000-09-04 02:18:13 +08:00
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2002-05-29 16:41:21 +08:00
|
|
|
#else /* Unix */
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API FILE *virtual_popen(const char *command, const char *type) /* {{{ */
|
2002-05-29 16:41:21 +08:00
|
|
|
{
|
2016-09-06 18:05:22 +08:00
|
|
|
size_t command_length;
|
2004-12-02 09:04:23 +08:00
|
|
|
int dir_length, extra = 0;
|
2002-05-29 16:41:21 +08:00
|
|
|
char *command_line;
|
2004-12-02 09:04:23 +08:00
|
|
|
char *ptr, *dir;
|
2002-05-29 16:41:21 +08:00
|
|
|
FILE *retval;
|
|
|
|
|
|
|
|
command_length = strlen(command);
|
|
|
|
|
2004-12-02 09:04:23 +08:00
|
|
|
dir_length = CWDG(cwd).cwd_length;
|
|
|
|
dir = CWDG(cwd).cwd;
|
|
|
|
while (dir_length > 0) {
|
|
|
|
if (*dir == '\'') extra+=3;
|
|
|
|
dir++;
|
|
|
|
dir_length--;
|
|
|
|
}
|
|
|
|
dir_length = CWDG(cwd).cwd_length;
|
|
|
|
dir = CWDG(cwd).cwd;
|
|
|
|
|
2013-10-17 16:40:43 +08:00
|
|
|
ptr = command_line = (char *) emalloc(command_length + sizeof("cd '' ; ") + dir_length + extra+1+1);
|
2002-05-29 16:41:21 +08:00
|
|
|
memcpy(ptr, "cd ", sizeof("cd ")-1);
|
|
|
|
ptr += sizeof("cd ")-1;
|
|
|
|
|
|
|
|
if (CWDG(cwd).cwd_length == 0) {
|
|
|
|
*ptr++ = DEFAULT_SLASH;
|
|
|
|
} else {
|
2004-12-02 09:04:23 +08:00
|
|
|
*ptr++ = '\'';
|
|
|
|
while (dir_length > 0) {
|
|
|
|
switch (*dir) {
|
|
|
|
case '\'':
|
|
|
|
*ptr++ = '\'';
|
|
|
|
*ptr++ = '\\';
|
|
|
|
*ptr++ = '\'';
|
|
|
|
/* fall-through */
|
|
|
|
default:
|
|
|
|
*ptr++ = *dir;
|
|
|
|
}
|
|
|
|
dir++;
|
|
|
|
dir_length--;
|
|
|
|
}
|
|
|
|
*ptr++ = '\'';
|
2002-05-29 16:41:21 +08:00
|
|
|
}
|
2010-09-10 22:02:19 +08:00
|
|
|
|
2002-05-29 16:41:21 +08:00
|
|
|
*ptr++ = ' ';
|
|
|
|
*ptr++ = ';';
|
|
|
|
*ptr++ = ' ';
|
|
|
|
|
|
|
|
memcpy(ptr, command, command_length+1);
|
|
|
|
retval = popen(command_line, type);
|
|
|
|
|
2013-10-17 16:40:43 +08:00
|
|
|
efree(command_line);
|
2002-05-29 16:41:21 +08:00
|
|
|
return retval;
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
#endif
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
CWD_API char *tsrm_realpath(const char *path, char *real_path) /* {{{ */
|
2006-02-09 04:50:03 +08:00
|
|
|
{
|
2006-11-10 20:59:27 +08:00
|
|
|
cwd_state new_state;
|
|
|
|
char cwd[MAXPATHLEN];
|
2006-02-09 04:50:03 +08:00
|
|
|
|
2007-07-10 21:21:11 +08:00
|
|
|
/* realpath("") returns CWD */
|
|
|
|
if (!*path) {
|
2013-10-17 16:40:43 +08:00
|
|
|
new_state.cwd = (char*)emalloc(1);
|
2007-07-10 21:21:11 +08:00
|
|
|
new_state.cwd[0] = '\0';
|
2010-09-10 22:02:19 +08:00
|
|
|
new_state.cwd_length = 0;
|
2010-08-27 18:09:52 +08:00
|
|
|
if (VCWD_GETCWD(cwd, MAXPATHLEN)) {
|
|
|
|
path = cwd;
|
2007-07-10 21:21:11 +08:00
|
|
|
}
|
2014-09-19 21:59:55 +08:00
|
|
|
} else if (!IS_ABSOLUTE_PATH(path, strlen(path)) &&
|
|
|
|
VCWD_GETCWD(cwd, MAXPATHLEN)) {
|
|
|
|
new_state.cwd = estrdup(cwd);
|
2017-07-27 05:10:07 +08:00
|
|
|
new_state.cwd_length = strlen(cwd);
|
2006-02-09 04:50:03 +08:00
|
|
|
} else {
|
2014-09-19 21:59:55 +08:00
|
|
|
new_state.cwd = (char*)emalloc(1);
|
|
|
|
new_state.cwd[0] = '\0';
|
|
|
|
new_state.cwd_length = 0;
|
2006-11-10 20:59:27 +08:00
|
|
|
}
|
|
|
|
|
2014-12-14 06:06:14 +08:00
|
|
|
if (virtual_file_ex(&new_state, path, NULL, CWD_REALPATH)) {
|
2013-10-17 16:40:43 +08:00
|
|
|
efree(new_state.cwd);
|
2006-11-10 20:59:27 +08:00
|
|
|
return NULL;
|
2006-02-09 04:50:03 +08:00
|
|
|
}
|
|
|
|
|
2006-11-10 20:59:27 +08:00
|
|
|
if (real_path) {
|
2017-07-27 05:10:07 +08:00
|
|
|
size_t copy_len = new_state.cwd_length>MAXPATHLEN-1 ? MAXPATHLEN-1 : new_state.cwd_length;
|
2006-11-10 20:59:27 +08:00
|
|
|
memcpy(real_path, new_state.cwd, copy_len);
|
|
|
|
real_path[copy_len] = '\0';
|
2013-10-17 16:40:43 +08:00
|
|
|
efree(new_state.cwd);
|
2006-11-10 20:59:27 +08:00
|
|
|
return real_path;
|
|
|
|
} else {
|
|
|
|
return new_state.cwd;
|
|
|
|
}
|
|
|
|
}
|
2007-08-10 17:09:46 +08:00
|
|
|
/* }}} */
|
2000-09-04 02:18:13 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Local variables:
|
|
|
|
* tab-width: 4
|
|
|
|
* c-basic-offset: 4
|
|
|
|
* End:
|
|
|
|
*/
|