Skip to content

json: Optimize escaping string in Encoder #133186

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

json: Optimize escaping string in Encoder #133186

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c2f1010
json: Optimize escaping string in Encoder
methane Apr 30, 2025
59e5131
add news and whatsnew
methane Apr 30, 2025
ee1a7f6
add comment
methane Apr 30, 2025
d026be3
apply suggested change
methane Apr 30, 2025
8e5e00b
use tmp buffer
methane May 1, 2025
19c0f1f
use UCS4 instead of UTF8
methane May 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Doc/whatsnew/3.14.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1518,6 +1518,13 @@ io
:gh:`120754` and :gh:`90102`.)


json
----

* Improve the performance of :class:`~json.JSONEncoder` encodes strings.
(Contributed by Inada Naoki in :gh:`133186`.)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should now retarget this change to Python 3.15 (move the text to Doc/whatsnew/3.15.rst).



uuid
----

Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Library/2025-04-30-07-27-30.gh-issue-133186.MSMobf.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Improve the performance of :class:`~json.JSONEncoder` encodes strings.
(Contributed by Inada Naoki in :gh:`133186`.)
152 changes: 119 additions & 33 deletions Modules/_json.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ typedef struct _PyEncoderObject {
char sort_keys;
char skipkeys;
int allow_nan;
PyCFunction fast_encode;
int (*fast_encode)(PyUnicodeWriter *, PyObject*);
} PyEncoderObject;

#define PyEncoderObject_CAST(op) ((PyEncoderObject *)(op))
Expand Down Expand Up @@ -102,8 +102,8 @@ static PyObject *
_encoded_const(PyObject *obj);
static void
raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end);
static PyObject *
encoder_encode_string(PyEncoderObject *s, PyObject *obj);
static int
encoder_write_string(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj);
static PyObject *
encoder_encode_float(PyEncoderObject *s, PyObject *obj);

Expand Down Expand Up @@ -303,6 +303,93 @@ escape_unicode(PyObject *pystr)
return rval;
}

// Take a PyUnicode pystr and write an ASCII-only escaped string to writer.
// Same to ascii_escape_unicode(), but write to PyUnicodeWriter instead of
// return Unicode object.
static int
write_escaped_ascii(PyUnicodeWriter *writer, PyObject *pystr)
{
Py_ssize_t i;
Py_ssize_t input_chars;
Py_ssize_t chars;
Py_ssize_t copy_len = 0;
const void *input;
int kind;
int ret;
unsigned char buf[12];

input_chars = PyUnicode_GET_LENGTH(pystr);
input = PyUnicode_DATA(pystr);
kind = PyUnicode_KIND(pystr);

ret = PyUnicodeWriter_WriteChar(writer, '"');
if (ret) return ret;

for (i = 0; i < input_chars; i++) {
Py_UCS4 c = PyUnicode_READ(kind, input, i);
if (S_CHAR(c)) {
copy_len++;
}
else {
ret = PyUnicodeWriter_WriteSubstring(writer, pystr, i-copy_len, i);
if (ret) return ret;
copy_len = 0;

chars = ascii_escape_unichar(c, buf, 0);
ret = PyUnicodeWriter_WriteUTF8(writer, (const char*)buf, chars);
if (ret) return ret;
}
}

ret = PyUnicodeWriter_WriteSubstring(writer, pystr, i-copy_len, i);
if (ret) return ret;

return PyUnicodeWriter_WriteChar(writer, '"');
}

// Take a PyUnicode pystr and write an escaped string to writer.
// Same to escape_unicode(), but write to PyUnicodeWriter instead of
// return Unicode object.
static int
write_escaped_unicode(PyUnicodeWriter *writer, PyObject *pystr)
{
Py_ssize_t i;
Py_ssize_t input_chars;
Py_ssize_t chars;
Py_ssize_t copy_len = 0;
const void *input;
int kind;
int ret;
unsigned char buf[12];

input_chars = PyUnicode_GET_LENGTH(pystr);
input = PyUnicode_DATA(pystr);
kind = PyUnicode_KIND(pystr);

ret = PyUnicodeWriter_WriteChar(writer, '"');
if (ret) return ret;

for (i = 0; i < input_chars; i++) {
Py_UCS4 c = PyUnicode_READ(kind, input, i);
if (c <= 0x1f || c == '\\' || c == '"') {
ret = PyUnicodeWriter_WriteSubstring(writer, pystr, i-copy_len, i);
if (ret) return ret;
copy_len = 0;

chars = ascii_escape_unichar(c, buf, 0);
ret = PyUnicodeWriter_WriteUTF8(writer, (const char*)buf, chars);
if (ret) return ret;
}
else {
copy_len++;
}
}

ret = PyUnicodeWriter_WriteSubstring(writer, pystr, i-copy_len, i);
if (ret) return ret;
return PyUnicodeWriter_WriteChar(writer, '"');
}

static void
raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end)
{
Expand Down Expand Up @@ -1255,8 +1342,11 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)

if (PyCFunction_Check(s->encoder)) {
PyCFunction f = PyCFunction_GetFunction(s->encoder);
if (f == py_encode_basestring_ascii || f == py_encode_basestring) {
s->fast_encode = f;
if (f == py_encode_basestring_ascii){
s->fast_encode = write_escaped_ascii;
}
else if (f == py_encode_basestring) {
s->fast_encode = write_escaped_unicode;
}
}

Expand Down Expand Up @@ -1437,33 +1527,35 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj)
return PyFloat_Type.tp_repr(obj);
}

static PyObject *
encoder_encode_string(PyEncoderObject *s, PyObject *obj)
static int
_steal_accumulate(PyUnicodeWriter *writer, PyObject *stolen)
{
/* Return the JSON representation of a string */
PyObject *encoded;
/* Append stolen and then decrement its reference count */
int rval = PyUnicodeWriter_WriteStr(writer, stolen);
Py_DECREF(stolen);
return rval;
}

static int
encoder_write_string(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj)
{
if (s->fast_encode) {
return s->fast_encode(NULL, obj);
return s->fast_encode(writer, obj);
}

/* Return the JSON representation of a string */
PyObject *encoded = PyObject_CallOneArg(s->encoder, obj);
if (encoded == NULL) {
return -1;
}
encoded = PyObject_CallOneArg(s->encoder, obj);
if (encoded != NULL && !PyUnicode_Check(encoded)) {
PyErr_Format(PyExc_TypeError,
"encoder() must return a string, not %.80s",
Py_TYPE(encoded)->tp_name);
Py_DECREF(encoded);
return NULL;
return -1;
}
return encoded;
}

static int
_steal_accumulate(PyUnicodeWriter *writer, PyObject *stolen)
{
/* Append stolen and then decrement its reference count */
int rval = PyUnicodeWriter_WriteStr(writer, stolen);
Py_DECREF(stolen);
return rval;
return _steal_accumulate(writer, encoded);
}

static int
Expand All @@ -1485,10 +1577,7 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer,
return PyUnicodeWriter_WriteUTF8(writer, "false", 5);
}
else if (PyUnicode_Check(obj)) {
PyObject *encoded = encoder_encode_string(s, obj);
if (encoded == NULL)
return -1;
return _steal_accumulate(writer, encoded);
return encoder_write_string(s, writer, obj);
}
else if (PyLong_Check(obj)) {
if (PyLong_CheckExact(obj)) {
Expand Down Expand Up @@ -1577,7 +1666,7 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs
PyObject *item_separator)
{
PyObject *keystr = NULL;
PyObject *encoded;
int rv;

if (PyUnicode_Check(key)) {
keystr = Py_NewRef(key);
Expand Down Expand Up @@ -1617,15 +1706,12 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs
}
}

encoded = encoder_encode_string(s, keystr);
rv = encoder_write_string(s, writer, keystr);
Py_DECREF(keystr);
if (encoded == NULL) {
return -1;
if (rv != 0) {
return rv;
}

if (_steal_accumulate(writer, encoded) < 0) {
return -1;
}
if (PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) {
return -1;
}
Expand Down
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy