From 65a15dce44eb75af57b87211712162072d2189ea Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Jun 2022 11:14:25 +0100 Subject: [PATCH 1/2] bpo-42658: Use LCMapStringEx in ntpath.normcase to match OS behaviour for case-folding (GH-93591) * bpo-42658: Use LCMapStringEx in ntpath.normcase to match OS behaviour for case-folding (GH-32010) * Use AsWideCharString to avoid memory leaks in deprectated unicode converter Co-authored-by: AN Long --- Lib/ntpath.py | 42 ++++++++-- Lib/test/test_ntpath.py | 2 + .../2022-03-20-15-47-35.bpo-42658.16eXtb.rst | 3 + Modules/_winapi.c | 79 +++++++++++++++++++ Modules/clinic/_winapi.c.h | 34 +++++++- 5 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2022-03-20-15-47-35.bpo-42658.16eXtb.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 527c7ae1938fbb..97edfa52aaafb8 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -23,6 +23,7 @@ import genericpath from genericpath import * + __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", @@ -41,14 +42,39 @@ def _get_bothseps(path): # Other normalizations (such as optimizing '../' away) are not done # (this is done by normpath). -def normcase(s): - """Normalize case of pathname. - - Makes all characters lowercase and all slashes into backslashes.""" - s = os.fspath(s) - if isinstance(s, bytes): - return s.replace(b'/', b'\\').lower() - else: +try: + from _winapi import ( + LCMapStringEx as _LCMapStringEx, + LOCALE_NAME_INVARIANT as _LOCALE_NAME_INVARIANT, + LCMAP_LOWERCASE as _LCMAP_LOWERCASE) + + def normcase(s): + """Normalize case of pathname. + + Makes all characters lowercase and all slashes into backslashes. + """ + s = os.fspath(s) + if not s: + return s + if isinstance(s, bytes): + encoding = sys.getfilesystemencoding() + s = s.decode(encoding, 'surrogateescape').replace('/', '\\') + s = _LCMapStringEx(_LOCALE_NAME_INVARIANT, + _LCMAP_LOWERCASE, s) + return s.encode(encoding, 'surrogateescape') + else: + return _LCMapStringEx(_LOCALE_NAME_INVARIANT, + _LCMAP_LOWERCASE, + s.replace('/', '\\')) +except ImportError: + def normcase(s): + """Normalize case of pathname. + + Makes all characters lowercase and all slashes into backslashes. + """ + s = os.fspath(s) + if isinstance(s, bytes): + return os.fsencode(os.fsdecode(s).replace('/', '\\').lower()) return s.replace('/', '\\').lower() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 661c59d6171635..8258b7e44dc8bf 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -807,6 +807,8 @@ def _check_function(self, func): def test_path_normcase(self): self._check_function(self.path.normcase) + if sys.platform == 'win32': + self.assertEqual(ntpath.normcase('\u03a9\u2126'), 'ωΩ') def test_path_isabs(self): self._check_function(self.path.isabs) diff --git a/Misc/NEWS.d/next/Windows/2022-03-20-15-47-35.bpo-42658.16eXtb.rst b/Misc/NEWS.d/next/Windows/2022-03-20-15-47-35.bpo-42658.16eXtb.rst new file mode 100644 index 00000000000000..852cc77676a31d --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2022-03-20-15-47-35.bpo-42658.16eXtb.rst @@ -0,0 +1,3 @@ +Support native Windows case-insensitive path comparisons by using +``LCMapStringEx`` instead of :func:`str.lower` in :func:`ntpath.normcase`. +Add ``LCMapStringEx`` to the :mod:`_winapi` module. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 3e24d512cac384..a3c30f23955450 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1512,6 +1512,68 @@ _winapi_PeekNamedPipe_impl(PyObject *module, HANDLE handle, int size) } } +/*[clinic input] +_winapi.LCMapStringEx + + locale: unicode + flags: DWORD + src: unicode + +[clinic start generated code]*/ + +static PyObject * +_winapi_LCMapStringEx_impl(PyObject *module, PyObject *locale, DWORD flags, + PyObject *src) +/*[clinic end generated code: output=8ea4c9d85a4a1f23 input=2fa6ebc92591731b]*/ +{ + if (flags & (LCMAP_SORTHANDLE | LCMAP_HASH | LCMAP_BYTEREV | + LCMAP_SORTKEY)) { + return PyErr_Format(PyExc_ValueError, "unsupported flags"); + } + + wchar_t *locale_ = PyUnicode_AsWideCharString(locale, NULL); + if (!locale_) { + return NULL; + } + wchar_t *src_ = PyUnicode_AsWideCharString(src, NULL); + if (!src_) { + PyMem_Free(locale_); + return NULL; + } + + int dest_size = LCMapStringEx(locale_, flags, src_, -1, NULL, 0, + NULL, NULL, 0); + if (dest_size == 0) { + PyMem_Free(locale_); + PyMem_Free(src_); + return PyErr_SetFromWindowsErr(0); + } + + wchar_t* dest = PyMem_NEW(wchar_t, dest_size); + if (dest == NULL) { + PyMem_Free(locale_); + PyMem_Free(src_); + return PyErr_NoMemory(); + } + + int nmapped = LCMapStringEx(locale_, flags, src_, -1, dest, dest_size, + NULL, NULL, 0); + if (nmapped == 0) { + DWORD error = GetLastError(); + PyMem_Free(locale_); + PyMem_Free(src_); + PyMem_DEL(dest); + return PyErr_SetFromWindowsErr(error); + } + + PyObject *ret = PyUnicode_FromWideChar(dest, dest_size - 1); + PyMem_Free(locale_); + PyMem_Free(src_); + PyMem_DEL(dest); + + return ret; +} + /*[clinic input] _winapi.ReadFile @@ -2023,6 +2085,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_OPENFILEMAPPING_METHODDEF _WINAPI_OPENPROCESS_METHODDEF _WINAPI_PEEKNAMEDPIPE_METHODDEF + _WINAPI_LCMAPSTRINGEX_METHODDEF _WINAPI_READFILE_METHODDEF _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF _WINAPI_TERMINATEPROCESS_METHODDEF @@ -2160,6 +2223,22 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, FILE_TYPE_PIPE); WINAPI_CONSTANT(F_DWORD, FILE_TYPE_REMOTE); + WINAPI_CONSTANT("u", LOCALE_NAME_INVARIANT); + WINAPI_CONSTANT(F_DWORD, LOCALE_NAME_MAX_LENGTH); + WINAPI_CONSTANT("u", LOCALE_NAME_SYSTEM_DEFAULT); + WINAPI_CONSTANT("u", LOCALE_NAME_USER_DEFAULT); + + WINAPI_CONSTANT(F_DWORD, LCMAP_FULLWIDTH); + WINAPI_CONSTANT(F_DWORD, LCMAP_HALFWIDTH); + WINAPI_CONSTANT(F_DWORD, LCMAP_HIRAGANA); + WINAPI_CONSTANT(F_DWORD, LCMAP_KATAKANA); + WINAPI_CONSTANT(F_DWORD, LCMAP_LINGUISTIC_CASING); + WINAPI_CONSTANT(F_DWORD, LCMAP_LOWERCASE); + WINAPI_CONSTANT(F_DWORD, LCMAP_SIMPLIFIED_CHINESE); + WINAPI_CONSTANT(F_DWORD, LCMAP_TITLECASE); + WINAPI_CONSTANT(F_DWORD, LCMAP_TRADITIONAL_CHINESE); + WINAPI_CONSTANT(F_DWORD, LCMAP_UPPERCASE); + WINAPI_CONSTANT("i", NULL); return 0; diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 5bda156d7aa80c..ac73789d799c87 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -840,6 +840,38 @@ _winapi_PeekNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(_winapi_LCMapStringEx__doc__, +"LCMapStringEx($module, /, locale, flags, src)\n" +"--\n" +"\n"); + +#define _WINAPI_LCMAPSTRINGEX_METHODDEF \ + {"LCMapStringEx", (PyCFunction)(void(*)(void))_winapi_LCMapStringEx, METH_FASTCALL|METH_KEYWORDS, _winapi_LCMapStringEx__doc__}, + +static PyObject * +_winapi_LCMapStringEx_impl(PyObject *module, PyObject *locale, DWORD flags, + PyObject *src); + +static PyObject * +_winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"locale", "flags", "src", NULL}; + static _PyArg_Parser _parser = {"UkU:LCMapStringEx", _keywords, 0}; + PyObject *locale; + DWORD flags; + PyObject *src; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &locale, &flags, &src)) { + goto exit; + } + return_value = _winapi_LCMapStringEx_impl(module, locale, flags, src); + +exit: + return return_value; +} + PyDoc_STRVAR(_winapi_ReadFile__doc__, "ReadFile($module, /, handle, size, overlapped=False)\n" "--\n" @@ -1184,4 +1216,4 @@ _winapi__mimetypes_read_windows_registry(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=ac3623be6e42017c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8e13179bf25bdea5 input=a9049054013a1b77]*/ From 67d038a72aa0aa5cc1919525c7810cd334847a87 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Jun 2022 16:42:32 +0100 Subject: [PATCH 2/2] Allow _winapi.LCMapStringEx to handle embedded nulls --- Modules/_winapi.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index a3c30f23955450..9b30a900326192 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1535,13 +1535,19 @@ _winapi_LCMapStringEx_impl(PyObject *module, PyObject *locale, DWORD flags, if (!locale_) { return NULL; } - wchar_t *src_ = PyUnicode_AsWideCharString(src, NULL); + Py_ssize_t srcLenAsSsize; + int srcLen; + wchar_t *src_ = PyUnicode_AsWideCharString(src, &srcLenAsSsize); if (!src_) { PyMem_Free(locale_); return NULL; } + srcLen = (int)srcLenAsSsize; + if (srcLen != srcLenAsSsize) { + srcLen = -1; + } - int dest_size = LCMapStringEx(locale_, flags, src_, -1, NULL, 0, + int dest_size = LCMapStringEx(locale_, flags, src_, srcLen, NULL, 0, NULL, NULL, 0); if (dest_size == 0) { PyMem_Free(locale_); @@ -1556,7 +1562,7 @@ _winapi_LCMapStringEx_impl(PyObject *module, PyObject *locale, DWORD flags, return PyErr_NoMemory(); } - int nmapped = LCMapStringEx(locale_, flags, src_, -1, dest, dest_size, + int nmapped = LCMapStringEx(locale_, flags, src_, srcLen, dest, dest_size, NULL, NULL, 0); if (nmapped == 0) { DWORD error = GetLastError(); @@ -1566,7 +1572,7 @@ _winapi_LCMapStringEx_impl(PyObject *module, PyObject *locale, DWORD flags, return PyErr_SetFromWindowsErr(error); } - PyObject *ret = PyUnicode_FromWideChar(dest, dest_size - 1); + PyObject *ret = PyUnicode_FromWideChar(dest, dest_size); PyMem_Free(locale_); PyMem_Free(src_); PyMem_DEL(dest); pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy