From c556f77d2abe34f7908877aa48ce2fbf294d8597 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 5 Apr 2020 02:35:11 +0900 Subject: [PATCH 1/5] bpo-1635741: Port _lzma module to multiphase initialization --- ...2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst | 1 + Modules/_lzmamodule.c | 551 +++++++++++------- Modules/clinic/_lzmamodule.c.h | 69 +-- 3 files changed, 365 insertions(+), 256 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst b/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst new file mode 100644 index 00000000000000..6937c0f2e48c7c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst @@ -0,0 +1 @@ +Port _lzma module to multiphase initialization. diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 2a62a683568505..d112e0fadf53bc 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -23,6 +23,20 @@ } } while (0) #define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) +typedef struct { + PyTypeObject *lzma_compressor_type; + PyTypeObject *lzma_decompressor_type; + PyObject *error_type; + PyObject *empty_tuple; +} _lzma_state; + +static inline _lzma_state* +get_lzma_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (_lzma_state *)state; +} /* Container formats: */ enum { @@ -56,17 +70,10 @@ typedef struct { PyThread_type_lock lock; } Decompressor; -/* LZMAError class object. */ -static PyObject *Error; - -/* An empty tuple, used by the filter specifier parsing code. */ -static PyObject *empty_tuple; - - /* Helper functions. */ static int -catch_lzma_error(lzma_ret lzret) +catch_lzma_error(_lzma_state *state, lzma_ret lzret) { switch (lzret) { case LZMA_OK: @@ -75,31 +82,31 @@ catch_lzma_error(lzma_ret lzret) case LZMA_STREAM_END: return 0; case LZMA_UNSUPPORTED_CHECK: - PyErr_SetString(Error, "Unsupported integrity check"); + PyErr_SetString(state->error_type, "Unsupported integrity check"); return 1; case LZMA_MEM_ERROR: PyErr_NoMemory(); return 1; case LZMA_MEMLIMIT_ERROR: - PyErr_SetString(Error, "Memory usage limit exceeded"); + PyErr_SetString(state->error_type, "Memory usage limit exceeded"); return 1; case LZMA_FORMAT_ERROR: - PyErr_SetString(Error, "Input format not supported by decoder"); + PyErr_SetString(state->error_type, "Input format not supported by decoder"); return 1; case LZMA_OPTIONS_ERROR: - PyErr_SetString(Error, "Invalid or unsupported options"); + PyErr_SetString(state->error_type, "Invalid or unsupported options"); return 1; case LZMA_DATA_ERROR: - PyErr_SetString(Error, "Corrupt input data"); + PyErr_SetString(state->error_type, "Corrupt input data"); return 1; case LZMA_BUF_ERROR: - PyErr_SetString(Error, "Insufficient buffer space"); + PyErr_SetString(state->error_type, "Insufficient buffer space"); return 1; case LZMA_PROG_ERROR: - PyErr_SetString(Error, "Internal error"); + PyErr_SetString(state->error_type, "Internal error"); return 1; default: - PyErr_Format(Error, "Unrecognized error from liblzma: %d", lzret); + PyErr_Format(state->error_type, "Unrecognized error from liblzma: %d", lzret); return 1; } } @@ -186,7 +193,7 @@ INT_TYPE_CONVERTER_FUNC(lzma_match_finder, lzma_mf_converter) the C lzma_filter structs expected by liblzma. */ static void * -parse_filter_spec_lzma(PyObject *spec) +parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) { static char *optnames[] = {"id", "preset", "dict_size", "lc", "lp", "pb", "mode", "nice_len", "mf", "depth", NULL}; @@ -217,11 +224,11 @@ parse_filter_spec_lzma(PyObject *spec) if (lzma_lzma_preset(options, preset)) { PyMem_Free(options); - PyErr_Format(Error, "Invalid compression preset: %u", preset); + PyErr_Format(state->error_type, "Invalid compression preset: %u", preset); return NULL; } - if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec, + if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OOO&O&O&O&O&O&O&O&", optnames, &id, &preset_obj, uint32_converter, &options->dict_size, @@ -235,20 +242,21 @@ parse_filter_spec_lzma(PyObject *spec) PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for LZMA filter"); PyMem_Free(options); - options = NULL; + return NULL; } + return options; } static void * -parse_filter_spec_delta(PyObject *spec) +parse_filter_spec_delta(_lzma_state *state, PyObject *spec) { static char *optnames[] = {"id", "dist", NULL}; PyObject *id; uint32_t dist = 1; lzma_options_delta *options; - if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec, "|OO&", optnames, + if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, &id, uint32_converter, &dist)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for delta filter"); @@ -264,14 +272,14 @@ parse_filter_spec_delta(PyObject *spec) } static void * -parse_filter_spec_bcj(PyObject *spec) +parse_filter_spec_bcj(_lzma_state *state, PyObject *spec) { static char *optnames[] = {"id", "start_offset", NULL}; PyObject *id; uint32_t start_offset = 0; lzma_options_bcj *options; - if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec, "|OO&", optnames, + if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, &id, uint32_converter, &start_offset)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for BCJ filter"); @@ -286,7 +294,7 @@ parse_filter_spec_bcj(PyObject *spec) } static int -lzma_filter_converter(PyObject *spec, void *ptr) +lzma_filter_converter(_lzma_state *state, PyObject *spec, void *ptr) { lzma_filter *f = (lzma_filter *)ptr; PyObject *id_obj; @@ -311,10 +319,10 @@ lzma_filter_converter(PyObject *spec, void *ptr) switch (f->id) { case LZMA_FILTER_LZMA1: case LZMA_FILTER_LZMA2: - f->options = parse_filter_spec_lzma(spec); + f->options = parse_filter_spec_lzma(state, spec); return f->options != NULL; case LZMA_FILTER_DELTA: - f->options = parse_filter_spec_delta(spec); + f->options = parse_filter_spec_delta(state, spec); return f->options != NULL; case LZMA_FILTER_X86: case LZMA_FILTER_POWERPC: @@ -322,7 +330,7 @@ lzma_filter_converter(PyObject *spec, void *ptr) case LZMA_FILTER_ARM: case LZMA_FILTER_ARMTHUMB: case LZMA_FILTER_SPARC: - f->options = parse_filter_spec_bcj(spec); + f->options = parse_filter_spec_bcj(state, spec); return f->options != NULL; default: PyErr_Format(PyExc_ValueError, "Invalid filter ID: %llu", f->id); @@ -340,7 +348,7 @@ free_filter_chain(lzma_filter filters[]) } static int -parse_filter_chain_spec(lzma_filter filters[], PyObject *filterspecs) +parse_filter_chain_spec(_lzma_state *state, lzma_filter filters[], PyObject *filterspecs) { Py_ssize_t i, num_filters; @@ -357,7 +365,7 @@ parse_filter_chain_spec(lzma_filter filters[], PyObject *filterspecs) for (i = 0; i < num_filters; i++) { int ok = 1; PyObject *spec = PySequence_GetItem(filterspecs, i); - if (spec == NULL || !lzma_filter_converter(spec, &filters[i])) + if (spec == NULL || !lzma_filter_converter(state, spec, &filters[i])) ok = 0; Py_XDECREF(spec); if (!ok) { @@ -492,6 +500,8 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) { Py_ssize_t data_size = 0; PyObject *result; + _lzma_state *state = PyType_GetModuleState(Py_TYPE(c)); + assert(state != NULL); result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE); if (result == NULL) @@ -509,7 +519,7 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) if (lzret == LZMA_BUF_ERROR && len == 0 && c->lzs.avail_out > 0) lzret = LZMA_OK; /* That wasn't a real error */ Py_END_ALLOW_THREADS - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) goto error; if ((action == LZMA_RUN && c->lzs.avail_in == 0) || (action == LZMA_FINISH && lzret == LZMA_STREAM_END)) { @@ -588,8 +598,8 @@ _lzma_LZMACompressor_flush_impl(Compressor *self) } static int -Compressor_init_xz(lzma_stream *lzs, int check, uint32_t preset, - PyObject *filterspecs) +Compressor_init_xz(_lzma_state *state, lzma_stream *lzs, + int check, uint32_t preset, PyObject *filterspecs) { lzma_ret lzret; @@ -598,19 +608,19 @@ Compressor_init_xz(lzma_stream *lzs, int check, uint32_t preset, } else { lzma_filter filters[LZMA_FILTERS_MAX + 1]; - if (parse_filter_chain_spec(filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) return -1; lzret = lzma_stream_encoder(lzs, filters, check); free_filter_chain(filters); } - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) return -1; else return 0; } static int -Compressor_init_alone(lzma_stream *lzs, uint32_t preset, PyObject *filterspecs) +Compressor_init_alone(_lzma_state *state, lzma_stream *lzs, uint32_t preset, PyObject *filterspecs) { lzma_ret lzret; @@ -618,14 +628,14 @@ Compressor_init_alone(lzma_stream *lzs, uint32_t preset, PyObject *filterspecs) lzma_options_lzma options; if (lzma_lzma_preset(&options, preset)) { - PyErr_Format(Error, "Invalid compression preset: %u", preset); + PyErr_Format(state->error_type, "Invalid compression preset: %u", preset); return -1; } lzret = lzma_alone_encoder(lzs, &options); } else { lzma_filter filters[LZMA_FILTERS_MAX + 1]; - if (parse_filter_chain_spec(filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) return -1; if (filters[0].id == LZMA_FILTER_LZMA1 && filters[1].id == LZMA_VLI_UNKNOWN) { @@ -638,14 +648,14 @@ Compressor_init_alone(lzma_stream *lzs, uint32_t preset, PyObject *filterspecs) } free_filter_chain(filters); } - if (PyErr_Occurred() || catch_lzma_error(lzret)) + if (PyErr_Occurred() || catch_lzma_error(state, lzret)) return -1; else return 0; } static int -Compressor_init_raw(lzma_stream *lzs, PyObject *filterspecs) +Compressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspecs) { lzma_filter filters[LZMA_FILTERS_MAX + 1]; lzma_ret lzret; @@ -655,11 +665,11 @@ Compressor_init_raw(lzma_stream *lzs, PyObject *filterspecs) "Must specify filters for FORMAT_RAW"); return -1; } - if (parse_filter_chain_spec(filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) return -1; lzret = lzma_raw_encoder(lzs, filters); free_filter_chain(filters); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) return -1; else return 0; @@ -706,7 +716,8 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) uint32_t preset = LZMA_PRESET_DEFAULT; PyObject *preset_obj = Py_None; PyObject *filterspecs = Py_None; - + _lzma_state *state = PyType_GetModuleState(Py_TYPE(self)); + assert(state != NULL); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiOO:LZMACompressor", arg_names, &format, &check, &preset_obj, @@ -745,17 +756,17 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) case FORMAT_XZ: if (check == -1) check = LZMA_CHECK_CRC64; - if (Compressor_init_xz(&self->lzs, check, preset, filterspecs) != 0) + if (Compressor_init_xz(state, &self->lzs, check, preset, filterspecs) != 0) break; return 0; case FORMAT_ALONE: - if (Compressor_init_alone(&self->lzs, preset, filterspecs) != 0) + if (Compressor_init_alone(state, &self->lzs, preset, filterspecs) != 0) break; return 0; case FORMAT_RAW: - if (Compressor_init_raw(&self->lzs, filterspecs) != 0) + if (Compressor_init_raw(state, &self->lzs, filterspecs) != 0) break; return 0; @@ -774,17 +785,42 @@ static void Compressor_dealloc(Compressor *self) { lzma_end(&self->lzs); - if (self->lock != NULL) + if (self->lock != NULL) { PyThread_free_lock(self->lock); - Py_TYPE(self)->tp_free((PyObject *)self); + } + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); +} + +/*[clinic input] +_lzma.LZMACompressor.__reduce__ +[clinic start generated code]*/ + +static PyObject * +_lzma_LZMACompressor___reduce___impl(Compressor *self) +/*[clinic end generated code: output=b49a0538d1cad752 input=6be52aba16b513c1]*/ +{ + PyErr_Format(PyExc_TypeError, + "cannot pickle %s object", + Py_TYPE(self)->tp_name); + return NULL; } static PyMethodDef Compressor_methods[] = { _LZMA_LZMACOMPRESSOR_COMPRESS_METHODDEF _LZMA_LZMACOMPRESSOR_FLUSH_METHODDEF + _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF {NULL} }; +static int +Compressor_traverse(Compressor *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + PyDoc_STRVAR(Compressor_doc, "LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None)\n" "\n" @@ -813,47 +849,26 @@ PyDoc_STRVAR(Compressor_doc, "\n" "For one-shot compression, use the compress() function instead.\n"); -static PyTypeObject Compressor_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_lzma.LZMACompressor", /* tp_name */ - sizeof(Compressor), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)Compressor_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - Compressor_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Compressor_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Compressor_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ +static PyType_Slot lzma_compressor_type_slots[] = { + {Py_tp_dealloc, Compressor_dealloc}, + {Py_tp_methods, Compressor_methods}, + {Py_tp_init, Compressor_init}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_doc, (char *)Compressor_doc}, + {Py_tp_traverse, Compressor_traverse}, + {0, 0} }; +static PyType_Spec lzma_compressor_type_spec = { + .name = "_lzma.LZMACompressor", + .basicsize = sizeof(Compressor), + // Calling PyType_GetModuleState() on a subclass is not safe. + // lzma_compressor_type_spec does not have Py_TPFLAGS_BASETYPE flag + // which prevents to create a subclass. + // So calling PyType_GetModuleState() in this file is always safe. + .flags = Py_TPFLAGS_DEFAULT, + .slots = lzma_compressor_type_slots, +}; /* LZMADecompressor class. */ @@ -867,6 +882,8 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) Py_ssize_t data_size = 0; PyObject *result; lzma_stream *lzs = &d->lzs; + _lzma_state *state = PyType_GetModuleState(Py_TYPE(d)); + assert(state != NULL); if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE) result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE); @@ -888,7 +905,7 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) lzret = LZMA_OK; /* That wasn't a real error */ Py_END_ALLOW_THREADS - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) goto error; if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK) d->check = lzma_get_check(&d->lzs); @@ -1082,16 +1099,16 @@ _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, } static int -Decompressor_init_raw(lzma_stream *lzs, PyObject *filterspecs) +Decompressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspecs) { lzma_filter filters[LZMA_FILTERS_MAX + 1]; lzma_ret lzret; - if (parse_filter_chain_spec(filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) return -1; lzret = lzma_raw_decoder(lzs, filters); free_filter_chain(filters); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) return -1; else return 0; @@ -1130,6 +1147,8 @@ _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, const uint32_t decoder_flags = LZMA_TELL_ANY_CHECK | LZMA_TELL_NO_CHECK; uint64_t memlimit_ = UINT64_MAX; lzma_ret lzret; + _lzma_state *state = PyType_GetModuleState(Py_TYPE(self)); + assert(state != NULL); if (memlimit != Py_None) { if (format == FORMAT_RAW) { @@ -1179,26 +1198,26 @@ _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, switch (format) { case FORMAT_AUTO: lzret = lzma_auto_decoder(&self->lzs, memlimit_, decoder_flags); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) break; return 0; case FORMAT_XZ: lzret = lzma_stream_decoder(&self->lzs, memlimit_, decoder_flags); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) break; return 0; case FORMAT_ALONE: self->check = LZMA_CHECK_NONE; lzret = lzma_alone_decoder(&self->lzs, memlimit_); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) break; return 0; case FORMAT_RAW: self->check = LZMA_CHECK_NONE; - if (Decompressor_init_raw(&self->lzs, filters) == -1) + if (Decompressor_init_raw(state, &self->lzs, filters) == -1) break; return 0; @@ -1223,13 +1242,38 @@ Decompressor_dealloc(Decompressor *self) lzma_end(&self->lzs); Py_CLEAR(self->unused_data); - if (self->lock != NULL) + if (self->lock != NULL) { PyThread_free_lock(self->lock); - Py_TYPE(self)->tp_free((PyObject *)self); + } + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); +} + +static int +Decompressor_traverse(Decompressor *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +/*[clinic input] +_lzma.LZMADecompressor.__reduce__ +[clinic start generated code]*/ + +static PyObject * +_lzma_LZMADecompressor___reduce___impl(Decompressor *self) +/*[clinic end generated code: output=2611fff0104a9c30 input=b9882e030aecd9a5]*/ +{ + PyErr_Format(PyExc_TypeError, + "cannot pickle %s object", + Py_TYPE(self)->tp_name); + return NULL; } static PyMethodDef Decompressor_methods[] = { _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF + _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF {NULL} }; @@ -1257,45 +1301,26 @@ static PyMemberDef Decompressor_members[] = { {NULL} }; -static PyTypeObject Decompressor_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_lzma.LZMADecompressor", /* tp_name */ - sizeof(Decompressor), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)Decompressor_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - _lzma_LZMADecompressor___init____doc__, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Decompressor_methods, /* tp_methods */ - Decompressor_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - _lzma_LZMADecompressor___init__, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ +static PyType_Slot lzma_decompressor_type_slots[] = { + {Py_tp_dealloc, Decompressor_dealloc}, + {Py_tp_methods, Decompressor_methods}, + {Py_tp_init, _lzma_LZMADecompressor___init__}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_doc, (char *)_lzma_LZMADecompressor___init____doc__}, + {Py_tp_traverse, Decompressor_traverse}, + {Py_tp_members, Decompressor_members}, + {0, 0} +}; + +static PyType_Spec lzma_decompressor_type_spec = { + .name = "_lzma.LZMADecompressor", + .basicsize = sizeof(Decompressor), + // Calling PyType_GetModuleState() on a subclass is not safe. + // lzma_decompressor_type_spec does not have Py_TPFLAGS_BASETYPE flag + // which prevents to create a subclass. + // So calling PyType_GetModuleState() in this file is always safe. + .flags = Py_TPFLAGS_DEFAULT, + .slots = lzma_decompressor_type_slots, }; @@ -1318,27 +1343,51 @@ _lzma_is_check_supported_impl(PyObject *module, int check_id) return PyBool_FromLong(lzma_check_is_supported(check_id)); } +PyDoc_STRVAR(_lzma__encode_filter_properties__doc__, +"_encode_filter_properties($module, filter, /)\n" +"--\n" +"\n" +"Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).\n" +"\n" +"The result does not include the filter ID itself, only the options."); -/*[clinic input] -_lzma._encode_filter_properties - filter: lzma_filter(c_default="{LZMA_VLI_UNKNOWN, NULL}") - / +#define _LZMA__ENCODE_FILTER_PROPERTIES_METHODDEF \ + {"_encode_filter_properties", (PyCFunction)_lzma__encode_filter_properties, METH_O, _lzma__encode_filter_properties__doc__}, -Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict). +static PyObject * +_lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter); -The result does not include the filter ID itself, only the options. -[clinic start generated code]*/ +static PyObject * +_lzma__encode_filter_properties(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + lzma_filter filter = {LZMA_VLI_UNKNOWN, NULL}; + _lzma_state *state = get_lzma_state(module); + assert(state != NULL); + if (!lzma_filter_converter(state, arg, &filter)) { + goto exit; + } + return_value = _lzma__encode_filter_properties_impl(module, filter); + +exit: + /* Cleanup for filter */ + if (filter.id != LZMA_VLI_UNKNOWN) + PyMem_Free(filter.options); + + return return_value; +} static PyObject * _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) -/*[clinic end generated code: output=5c93c8e14e7be5a8 input=d4c64f1b557c77d4]*/ { lzma_ret lzret; uint32_t encoded_size; PyObject *result = NULL; + _lzma_state *state = get_lzma_state(module); + assert(state != NULL); lzret = lzma_properties_size(&encoded_size, &filter); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) goto error; result = PyBytes_FromStringAndSize(NULL, encoded_size); @@ -1347,7 +1396,7 @@ _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) lzret = lzma_properties_encode( &filter, (uint8_t *)PyBytes_AS_STRING(result)); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) goto error; return result; @@ -1378,10 +1427,12 @@ _lzma__decode_filter_properties_impl(PyObject *module, lzma_vli filter_id, lzma_ret lzret; PyObject *result = NULL; filter.id = filter_id; + _lzma_state *state = get_lzma_state(module); + assert(state != NULL); lzret = lzma_properties_decode( &filter, NULL, encoded_props->buf, encoded_props->len); - if (catch_lzma_error(lzret)) + if (catch_lzma_error(state, lzret)) return NULL; result = build_filter_spec(&filter); @@ -1392,28 +1443,6 @@ _lzma__decode_filter_properties_impl(PyObject *module, lzma_vli filter_id, return result; } - -/* Module initialization. */ - -static PyMethodDef module_methods[] = { - _LZMA_IS_CHECK_SUPPORTED_METHODDEF - _LZMA__ENCODE_FILTER_PROPERTIES_METHODDEF - _LZMA__DECODE_FILTER_PROPERTIES_METHODDEF - {NULL} -}; - -static PyModuleDef _lzmamodule = { - PyModuleDef_HEAD_INIT, - "_lzma", - NULL, - -1, - module_methods, - NULL, - NULL, - NULL, - NULL, -}; - /* Some of our constants are more than 32 bits wide, so PyModule_AddIntConstant would not work correctly on platforms with 32-bit longs. */ static int @@ -1428,67 +1457,145 @@ module_add_int_constant(PyObject *m, const char *name, long long value) return -1; } -#define ADD_INT_PREFIX_MACRO(m, macro) \ - module_add_int_constant(m, #macro, LZMA_ ## macro) - -PyMODINIT_FUNC -PyInit__lzma(void) +static int +lzma_exec(PyObject *module) { - PyObject *m; +#define ADD_INT_PREFIX_MACRO(module, macro) \ + do { \ + if (module_add_int_constant(module, #macro, LZMA_ ## macro) < 0) { \ + return -1; \ + } \ + } while(0) + +#define ADD_INT_MACRO(module, macro) \ + do { \ + if (PyModule_AddIntMacro(module, macro) < 0) { \ + return -1; \ + } \ + } while (0) - empty_tuple = PyTuple_New(0); - if (empty_tuple == NULL) - return NULL; - m = PyModule_Create(&_lzmamodule); - if (m == NULL) - return NULL; + _lzma_state *state = get_lzma_state(module); - if (PyModule_AddIntMacro(m, FORMAT_AUTO) == -1 || - PyModule_AddIntMacro(m, FORMAT_XZ) == -1 || - PyModule_AddIntMacro(m, FORMAT_ALONE) == -1 || - PyModule_AddIntMacro(m, FORMAT_RAW) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_NONE) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_CRC32) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_CRC64) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_SHA256) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_ID_MAX) == -1 || - ADD_INT_PREFIX_MACRO(m, CHECK_UNKNOWN) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_LZMA1) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_LZMA2) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_DELTA) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_X86) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_IA64) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_ARM) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_ARMTHUMB) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_SPARC) == -1 || - ADD_INT_PREFIX_MACRO(m, FILTER_POWERPC) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_HC3) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_HC4) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_BT2) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_BT3) == -1 || - ADD_INT_PREFIX_MACRO(m, MF_BT4) == -1 || - ADD_INT_PREFIX_MACRO(m, MODE_FAST) == -1 || - ADD_INT_PREFIX_MACRO(m, MODE_NORMAL) == -1 || - ADD_INT_PREFIX_MACRO(m, PRESET_DEFAULT) == -1 || - ADD_INT_PREFIX_MACRO(m, PRESET_EXTREME) == -1) - return NULL; + state->empty_tuple = PyTuple_New(0); + if (state->empty_tuple == NULL) { + return -1; + } - Error = PyErr_NewExceptionWithDoc( - "_lzma.LZMAError", "Call to liblzma failed.", NULL, NULL); - if (Error == NULL) - return NULL; - Py_INCREF(Error); - if (PyModule_AddObject(m, "LZMAError", Error) == -1) - return NULL; + ADD_INT_MACRO(module, FORMAT_AUTO); + ADD_INT_MACRO(module, FORMAT_XZ); + ADD_INT_MACRO(module, FORMAT_ALONE); + ADD_INT_MACRO(module, FORMAT_RAW); + ADD_INT_PREFIX_MACRO(module, CHECK_NONE); + ADD_INT_PREFIX_MACRO(module, CHECK_CRC32); + ADD_INT_PREFIX_MACRO(module, CHECK_CRC64); + ADD_INT_PREFIX_MACRO(module, CHECK_SHA256); + ADD_INT_PREFIX_MACRO(module, CHECK_ID_MAX); + ADD_INT_PREFIX_MACRO(module, CHECK_UNKNOWN); + ADD_INT_PREFIX_MACRO(module, FILTER_LZMA1); + ADD_INT_PREFIX_MACRO(module, FILTER_LZMA2); + ADD_INT_PREFIX_MACRO(module, FILTER_DELTA); + ADD_INT_PREFIX_MACRO(module, FILTER_X86); + ADD_INT_PREFIX_MACRO(module, FILTER_IA64); + ADD_INT_PREFIX_MACRO(module, FILTER_ARM); + ADD_INT_PREFIX_MACRO(module, FILTER_ARMTHUMB); + ADD_INT_PREFIX_MACRO(module, FILTER_SPARC); + ADD_INT_PREFIX_MACRO(module, FILTER_POWERPC); + ADD_INT_PREFIX_MACRO(module, MF_HC3); + ADD_INT_PREFIX_MACRO(module, MF_HC4); + ADD_INT_PREFIX_MACRO(module, MF_BT2); + ADD_INT_PREFIX_MACRO(module, MF_BT3); + ADD_INT_PREFIX_MACRO(module, MF_BT4); + ADD_INT_PREFIX_MACRO(module, MODE_FAST); + ADD_INT_PREFIX_MACRO(module, MODE_NORMAL); + ADD_INT_PREFIX_MACRO(module, PRESET_DEFAULT); + ADD_INT_PREFIX_MACRO(module, PRESET_EXTREME); + + + state->error_type = PyErr_NewExceptionWithDoc("_lzma.LZMAError", "Call to liblzma failed.", NULL, NULL); + if (state->error_type == NULL) { + return -1; + } - if (PyModule_AddType(m, &Compressor_type) < 0) { - return NULL; + if (PyModule_AddType(module, (PyTypeObject *)state->error_type) < 0) { + return -1; } - if (PyModule_AddType(m, &Decompressor_type) < 0) { - return NULL; + + state->lzma_compressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &lzma_compressor_type_spec, NULL); + if (state->lzma_compressor_type == NULL) { + return -1; + } + + if (PyModule_AddType(module, state->lzma_compressor_type) < 0) { + return -1; + } + + state->lzma_decompressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &lzma_decompressor_type_spec, NULL); + if (state->lzma_decompressor_type == NULL) { + return -1; + } + + if (PyModule_AddType(module, state->lzma_decompressor_type) < 0) { + return -1; } - return m; + return 0; +} + +static PyMethodDef lzma_methods[] = { + _LZMA_IS_CHECK_SUPPORTED_METHODDEF + _LZMA__ENCODE_FILTER_PROPERTIES_METHODDEF + _LZMA__DECODE_FILTER_PROPERTIES_METHODDEF + {NULL} +}; + +static PyModuleDef_Slot lzma_slots[] = { + {Py_mod_exec, lzma_exec}, + {0, NULL} +}; + +static int +lzma_traverse(PyObject *module, visitproc visit, void *arg) +{ + _lzma_state *state = get_lzma_state(module); + Py_VISIT(state->lzma_compressor_type); + Py_VISIT(state->lzma_decompressor_type); + Py_VISIT(state->empty_tuple); + return 0; +} + +static int +lzma_clear(PyObject *module) +{ + _lzma_state *state = get_lzma_state(module); + Py_CLEAR(state->lzma_compressor_type); + Py_CLEAR(state->lzma_decompressor_type); + Py_CLEAR(state->empty_tuple); + return 0; +} + +static void +lzma_free(void *module) +{ + lzma_clear((PyObject *)module); +} + +static PyModuleDef _lzmamodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_lzma", + .m_size = sizeof(_lzma_state), + .m_methods = lzma_methods, + .m_slots = lzma_slots, + .m_traverse = lzma_traverse, + .m_clear = lzma_clear, + .m_free = lzma_free, +}; + +PyMODINIT_FUNC +PyInit__lzma(void) +{ + return PyModuleDef_Init(&_lzmamodule); } diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index e4e0a7945a8fba..e15cc0c7e743b6 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -65,6 +65,23 @@ _lzma_LZMACompressor_flush(Compressor *self, PyObject *Py_UNUSED(ignored)) return _lzma_LZMACompressor_flush_impl(self); } +PyDoc_STRVAR(_lzma_LZMACompressor___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)_lzma_LZMACompressor___reduce__, METH_NOARGS, _lzma_LZMACompressor___reduce____doc__}, + +static PyObject * +_lzma_LZMACompressor___reduce___impl(Compressor *self); + +static PyObject * +_lzma_LZMACompressor___reduce__(Compressor *self, PyObject *Py_UNUSED(ignored)) +{ + return _lzma_LZMACompressor___reduce___impl(self); +} + PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__, "decompress($self, /, data, max_length=-1)\n" "--\n" @@ -211,6 +228,23 @@ _lzma_LZMADecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs return return_value; } +PyDoc_STRVAR(_lzma_LZMADecompressor___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)_lzma_LZMADecompressor___reduce__, METH_NOARGS, _lzma_LZMADecompressor___reduce____doc__}, + +static PyObject * +_lzma_LZMADecompressor___reduce___impl(Decompressor *self); + +static PyObject * +_lzma_LZMADecompressor___reduce__(Decompressor *self, PyObject *Py_UNUSED(ignored)) +{ + return _lzma_LZMADecompressor___reduce___impl(self); +} + PyDoc_STRVAR(_lzma_is_check_supported__doc__, "is_check_supported($module, check_id, /)\n" "--\n" @@ -241,39 +275,6 @@ _lzma_is_check_supported(PyObject *module, PyObject *arg) return return_value; } -PyDoc_STRVAR(_lzma__encode_filter_properties__doc__, -"_encode_filter_properties($module, filter, /)\n" -"--\n" -"\n" -"Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).\n" -"\n" -"The result does not include the filter ID itself, only the options."); - -#define _LZMA__ENCODE_FILTER_PROPERTIES_METHODDEF \ - {"_encode_filter_properties", (PyCFunction)_lzma__encode_filter_properties, METH_O, _lzma__encode_filter_properties__doc__}, - -static PyObject * -_lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter); - -static PyObject * -_lzma__encode_filter_properties(PyObject *module, PyObject *arg) -{ - PyObject *return_value = NULL; - lzma_filter filter = {LZMA_VLI_UNKNOWN, NULL}; - - if (!lzma_filter_converter(arg, &filter)) { - goto exit; - } - return_value = _lzma__encode_filter_properties_impl(module, filter); - -exit: - /* Cleanup for filter */ - if (filter.id != LZMA_VLI_UNKNOWN) - PyMem_Free(filter.options); - - return return_value; -} - PyDoc_STRVAR(_lzma__decode_filter_properties__doc__, "_decode_filter_properties($module, filter_id, encoded_props, /)\n" "--\n" @@ -319,4 +320,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=d6e997ebc269f78f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d89b6159e98544be input=a9049054013a1b77]*/ From 14ef8fa7510a2b413b070526d1f9cab5a60dac84 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 22 Jun 2020 23:45:43 +0900 Subject: [PATCH 2/5] bpo-1635741: Fix memory leak --- Modules/_lzmamodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index d112e0fadf53bc..40eabb89ab48e3 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1563,6 +1563,7 @@ lzma_traverse(PyObject *module, visitproc visit, void *arg) _lzma_state *state = get_lzma_state(module); Py_VISIT(state->lzma_compressor_type); Py_VISIT(state->lzma_decompressor_type); + Py_VISIT(state->error_type); Py_VISIT(state->empty_tuple); return 0; } @@ -1573,6 +1574,7 @@ lzma_clear(PyObject *module) _lzma_state *state = get_lzma_state(module); Py_CLEAR(state->lzma_compressor_type); Py_CLEAR(state->lzma_decompressor_type); + Py_CLEAR(state->error_type); Py_CLEAR(state->empty_tuple); return 0; } From 304199581b9102ec16343b5cc6ab2986442e3da2 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 22 Jun 2020 23:47:00 +0900 Subject: [PATCH 3/5] bpo-1635741: Update NEWS.d --- .../2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst b/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst index 6937c0f2e48c7c..956d0b68a8dfb1 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-04-05-02-35-08.bpo-1635741.Kfe9fT.rst @@ -1 +1 @@ -Port _lzma module to multiphase initialization. +Port :mod:`_lzma` to multiphase initialization. From 60d315c31a92666f777a4ff92b184e676917dcc6 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 22 Jun 2020 23:55:57 +0900 Subject: [PATCH 4/5] bpo-1635741: Apply PEP 7 --- Modules/_lzmamodule.c | 179 ++++++++++++++++++++++++++++-------------- 1 file changed, 118 insertions(+), 61 deletions(-) diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 40eabb89ab48e3..89936e698b76b7 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -114,8 +114,9 @@ catch_lzma_error(_lzma_state *state, lzma_ret lzret) static void* PyLzma_Malloc(void *opaque, size_t items, size_t size) { - if (size != 0 && items > (size_t)PY_SSIZE_T_MAX / size) + if (size != 0 && items > (size_t)PY_SSIZE_T_MAX / size) { return NULL; + } /* PyMem_Malloc() cannot be used: the GIL is not held when lzma_code() is called */ return PyMem_RawMalloc(items * size); @@ -139,8 +140,9 @@ grow_buffer(PyObject **buf, Py_ssize_t max_length) Py_ssize_t size = PyBytes_GET_SIZE(*buf); Py_ssize_t newsize = size + (size >> 3) + 6; - if (max_length > 0 && newsize > max_length) + if (max_length > 0 && newsize > max_length) { newsize = max_length; + } return _PyBytes_Resize(buf, newsize); } @@ -207,20 +209,24 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) preset_obj = PyMapping_GetItemString(spec, "preset"); if (preset_obj == NULL) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) + if (PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); - else + } + else { return NULL; + } } else { int ok = uint32_converter(preset_obj, &preset); Py_DECREF(preset_obj); - if (!ok) + if (!ok) { return NULL; + } } options = (lzma_options_lzma *)PyMem_Calloc(1, sizeof *options); - if (options == NULL) + if (options == NULL) { return PyErr_NoMemory(); + } if (lzma_lzma_preset(options, preset)) { PyMem_Free(options); @@ -264,8 +270,9 @@ parse_filter_spec_delta(_lzma_state *state, PyObject *spec) } options = (lzma_options_delta *)PyMem_Calloc(1, sizeof *options); - if (options == NULL) + if (options == NULL) { return PyErr_NoMemory(); + } options->type = LZMA_DELTA_TYPE_BYTE; options->dist = dist; return options; @@ -287,8 +294,9 @@ parse_filter_spec_bcj(_lzma_state *state, PyObject *spec) } options = (lzma_options_bcj *)PyMem_Calloc(1, sizeof *options); - if (options == NULL) + if (options == NULL) { return PyErr_NoMemory(); + } options->start_offset = start_offset; return options; } @@ -313,8 +321,9 @@ lzma_filter_converter(_lzma_state *state, PyObject *spec, void *ptr) } f->id = PyLong_AsUnsignedLongLong(id_obj); Py_DECREF(id_obj); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { return 0; + } switch (f->id) { case LZMA_FILTER_LZMA1: @@ -341,10 +350,9 @@ lzma_filter_converter(_lzma_state *state, PyObject *spec, void *ptr) static void free_filter_chain(lzma_filter filters[]) { - int i; - - for (i = 0; filters[i].id != LZMA_VLI_UNKNOWN; i++) + for (int i = 0; filters[i].id != LZMA_VLI_UNKNOWN; i++) { PyMem_Free(filters[i].options); + } } static int @@ -353,8 +361,9 @@ parse_filter_chain_spec(_lzma_state *state, lzma_filter filters[], PyObject *fil Py_ssize_t i, num_filters; num_filters = PySequence_Length(filterspecs); - if (num_filters == -1) + if (num_filters == -1) { return -1; + } if (num_filters > LZMA_FILTERS_MAX) { PyErr_Format(PyExc_ValueError, "Too many filters - liblzma supports a maximum of %d", @@ -365,8 +374,9 @@ parse_filter_chain_spec(_lzma_state *state, lzma_filter filters[], PyObject *fil for (i = 0; i < num_filters; i++) { int ok = 1; PyObject *spec = PySequence_GetItem(filterspecs, i); - if (spec == NULL || !lzma_filter_converter(state, spec, &filters[i])) + if (spec == NULL || !lzma_filter_converter(state, spec, &filters[i])) { ok = 0; + } Py_XDECREF(spec); if (!ok) { filters[i].id = LZMA_VLI_UNKNOWN; @@ -391,8 +401,9 @@ spec_add_field(PyObject *spec, _Py_Identifier *key, unsigned long long value) PyObject *value_object; value_object = PyLong_FromUnsignedLongLong(value); - if (value_object == NULL) + if (value_object == NULL) { return -1; + } status = _PyDict_SetItemId(spec, key, value_object); Py_DECREF(value_object); @@ -405,8 +416,9 @@ build_filter_spec(const lzma_filter *f) PyObject *spec; spec = PyDict_New(); - if (spec == NULL) + if (spec == NULL) { return NULL; + } #define ADD_FIELD(SOURCE, FIELD) \ do { \ @@ -504,8 +516,9 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) assert(state != NULL); result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE); - if (result == NULL) + if (result == NULL) { return NULL; + } c->lzs.next_in = data; c->lzs.avail_in = len; c->lzs.next_out = (uint8_t *)PyBytes_AS_STRING(result); @@ -516,11 +529,13 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) Py_BEGIN_ALLOW_THREADS lzret = lzma_code(&c->lzs, action); data_size = (char *)c->lzs.next_out - PyBytes_AS_STRING(result); - if (lzret == LZMA_BUF_ERROR && len == 0 && c->lzs.avail_out > 0) + if (lzret == LZMA_BUF_ERROR && len == 0 && c->lzs.avail_out > 0) { lzret = LZMA_OK; /* That wasn't a real error */ + } Py_END_ALLOW_THREADS - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { goto error; + } if ((action == LZMA_RUN && c->lzs.avail_in == 0) || (action == LZMA_FINISH && lzret == LZMA_STREAM_END)) { break; @@ -532,8 +547,9 @@ compress(Compressor *c, uint8_t *data, size_t len, lzma_action action) } } if (data_size != PyBytes_GET_SIZE(result)) - if (_PyBytes_Resize(&result, data_size) == -1) + if (_PyBytes_Resize(&result, data_size) == -1) { goto error; + } return result; error: @@ -562,10 +578,12 @@ _lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data) PyObject *result = NULL; ACQUIRE_LOCK(self); - if (self->flushed) + if (self->flushed) { PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); - else + } + else { result = compress(self, data->buf, data->len, LZMA_RUN); + } RELEASE_LOCK(self); return result; } @@ -613,10 +631,12 @@ Compressor_init_xz(_lzma_state *state, lzma_stream *lzs, lzret = lzma_stream_encoder(lzs, filters, check); free_filter_chain(filters); } - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { return -1; - else + } + else { return 0; + } } static int @@ -648,10 +668,12 @@ Compressor_init_alone(_lzma_state *state, lzma_stream *lzs, uint32_t preset, PyO } free_filter_chain(filters); } - if (PyErr_Occurred() || catch_lzma_error(state, lzret)) + if (PyErr_Occurred() || catch_lzma_error(state, lzret)) { return -1; - else + } + else { return 0; + } } static int @@ -665,14 +687,17 @@ Compressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspecs) "Must specify filters for FORMAT_RAW"); return -1; } - if (parse_filter_chain_spec(state, filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) { return -1; + } lzret = lzma_raw_encoder(lzs, filters); free_filter_chain(filters); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { return -1; - else + } + else { return 0; + } } /*[-clinic input] @@ -721,8 +746,9 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiOO:LZMACompressor", arg_names, &format, &check, &preset_obj, - &filterspecs)) + &filterspecs)) { return -1; + } if (format != FORMAT_XZ && check != -1 && check != LZMA_CHECK_NONE) { PyErr_SetString(PyExc_ValueError, @@ -736,9 +762,11 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) return -1; } - if (preset_obj != Py_None) - if (!uint32_converter(preset_obj, &preset)) + if (preset_obj != Py_None) { + if (!uint32_converter(preset_obj, &preset)) { return -1; + } + } self->alloc.opaque = NULL; self->alloc.alloc = PyLzma_Malloc; @@ -754,20 +782,24 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) self->flushed = 0; switch (format) { case FORMAT_XZ: - if (check == -1) + if (check == -1) { check = LZMA_CHECK_CRC64; - if (Compressor_init_xz(state, &self->lzs, check, preset, filterspecs) != 0) + } + if (Compressor_init_xz(state, &self->lzs, check, preset, filterspecs) != 0) { break; + } return 0; case FORMAT_ALONE: - if (Compressor_init_alone(state, &self->lzs, preset, filterspecs) != 0) + if (Compressor_init_alone(state, &self->lzs, preset, filterspecs) != 0) { break; + } return 0; case FORMAT_RAW: - if (Compressor_init_raw(state, &self->lzs, filterspecs) != 0) + if (Compressor_init_raw(state, &self->lzs, filterspecs) != 0) { break; + } return 0; default: @@ -885,12 +917,15 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) _lzma_state *state = PyType_GetModuleState(Py_TYPE(d)); assert(state != NULL); - if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE) + if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE) { result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE); - else + } + else { result = PyBytes_FromStringAndSize(NULL, max_length); - if (result == NULL) + } + if (result == NULL) { return NULL; + } lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result); lzs->avail_out = PyBytes_GET_SIZE(result); @@ -901,14 +936,17 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) Py_BEGIN_ALLOW_THREADS lzret = lzma_code(lzs, LZMA_RUN); data_size = (char *)lzs->next_out - PyBytes_AS_STRING(result); - if (lzret == LZMA_BUF_ERROR && lzs->avail_in == 0 && lzs->avail_out > 0) + if (lzret == LZMA_BUF_ERROR && lzs->avail_in == 0 && lzs->avail_out > 0) { lzret = LZMA_OK; /* That wasn't a real error */ + } Py_END_ALLOW_THREADS - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { goto error; - if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK) + } + if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK) { d->check = lzma_get_check(&d->lzs); + } if (lzret == LZMA_STREAM_END) { d->eof = 1; break; @@ -917,19 +955,23 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) Maybe lzs's internal state still have a few bytes can be output, grow the output buffer and continue if max_lengh < 0. */ - if (data_size == max_length) + if (data_size == max_length) { break; - if (grow_buffer(&result, max_length) == -1) + } + if (grow_buffer(&result, max_length) == -1) { goto error; + } lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result) + data_size; lzs->avail_out = PyBytes_GET_SIZE(result) - data_size; } else if (lzs->avail_in == 0) { break; } } - if (data_size != PyBytes_GET_SIZE(result)) - if (_PyBytes_Resize(&result, data_size) == -1) + if (data_size != PyBytes_GET_SIZE(result)) { + if (_PyBytes_Resize(&result, data_size) == -1) { goto error; + } + } return result; @@ -1001,8 +1043,9 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) if (lzs->avail_in > 0) { Py_XSETREF(d->unused_data, PyBytes_FromStringAndSize((char *)lzs->next_in, lzs->avail_in)); - if (d->unused_data == NULL) + if (d->unused_data == NULL) { goto error; + } } } else if (lzs->avail_in == 0) { @@ -1104,14 +1147,17 @@ Decompressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspec lzma_filter filters[LZMA_FILTERS_MAX + 1]; lzma_ret lzret; - if (parse_filter_chain_spec(state, filters, filterspecs) == -1) + if (parse_filter_chain_spec(state, filters, filterspecs) == -1) { return -1; + } lzret = lzma_raw_decoder(lzs, filters); free_filter_chain(filters); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { return -1; - else + } + else { return 0; + } } /*[clinic input] @@ -1157,8 +1203,9 @@ _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, return -1; } memlimit_ = PyLong_AsUnsignedLongLong(memlimit); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { return -1; + } } if (format == FORMAT_RAW && filters == Py_None) { @@ -1192,33 +1239,38 @@ _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, self->input_buffer = NULL; self->input_buffer_size = 0; Py_XSETREF(self->unused_data, PyBytes_FromStringAndSize(NULL, 0)); - if (self->unused_data == NULL) + if (self->unused_data == NULL) { goto error; + } switch (format) { case FORMAT_AUTO: lzret = lzma_auto_decoder(&self->lzs, memlimit_, decoder_flags); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { break; + } return 0; case FORMAT_XZ: lzret = lzma_stream_decoder(&self->lzs, memlimit_, decoder_flags); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { break; + } return 0; case FORMAT_ALONE: self->check = LZMA_CHECK_NONE; lzret = lzma_alone_decoder(&self->lzs, memlimit_); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { break; + } return 0; case FORMAT_RAW: self->check = LZMA_CHECK_NONE; - if (Decompressor_init_raw(state, &self->lzs, filters) == -1) + if (Decompressor_init_raw(state, &self->lzs, filters) == -1) { break; + } return 0; default: @@ -1371,8 +1423,9 @@ _lzma__encode_filter_properties(PyObject *module, PyObject *arg) exit: /* Cleanup for filter */ - if (filter.id != LZMA_VLI_UNKNOWN) + if (filter.id != LZMA_VLI_UNKNOWN) { PyMem_Free(filter.options); + } return return_value; } @@ -1396,8 +1449,9 @@ _lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter) lzret = lzma_properties_encode( &filter, (uint8_t *)PyBytes_AS_STRING(result)); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { goto error; + } return result; @@ -1432,8 +1486,9 @@ _lzma__decode_filter_properties_impl(PyObject *module, lzma_vli filter_id, lzret = lzma_properties_decode( &filter, NULL, encoded_props->buf, encoded_props->len); - if (catch_lzma_error(state, lzret)) + if (catch_lzma_error(state, lzret)) { return NULL; + } result = build_filter_spec(&filter); @@ -1449,10 +1504,12 @@ static int module_add_int_constant(PyObject *m, const char *name, long long value) { PyObject *o = PyLong_FromLongLong(value); - if (o == NULL) + if (o == NULL) { return -1; - if (PyModule_AddObject(m, name, o) == 0) + } + if (PyModule_AddObject(m, name, o) == 0) { return 0; + } Py_DECREF(o); return -1; } From f60b5912f610401af5f1fa2185104ead376c8ff7 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Tue, 23 Jun 2020 00:30:07 +0900 Subject: [PATCH 5/5] bpo-1635741: Apply Victor's review --- Modules/_lzmamodule.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 89936e698b76b7..24e1d6c2884ce5 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -26,7 +26,7 @@ typedef struct { PyTypeObject *lzma_compressor_type; PyTypeObject *lzma_decompressor_type; - PyObject *error_type; + PyObject *error; PyObject *empty_tuple; } _lzma_state; @@ -82,31 +82,31 @@ catch_lzma_error(_lzma_state *state, lzma_ret lzret) case LZMA_STREAM_END: return 0; case LZMA_UNSUPPORTED_CHECK: - PyErr_SetString(state->error_type, "Unsupported integrity check"); + PyErr_SetString(state->error, "Unsupported integrity check"); return 1; case LZMA_MEM_ERROR: PyErr_NoMemory(); return 1; case LZMA_MEMLIMIT_ERROR: - PyErr_SetString(state->error_type, "Memory usage limit exceeded"); + PyErr_SetString(state->error, "Memory usage limit exceeded"); return 1; case LZMA_FORMAT_ERROR: - PyErr_SetString(state->error_type, "Input format not supported by decoder"); + PyErr_SetString(state->error, "Input format not supported by decoder"); return 1; case LZMA_OPTIONS_ERROR: - PyErr_SetString(state->error_type, "Invalid or unsupported options"); + PyErr_SetString(state->error, "Invalid or unsupported options"); return 1; case LZMA_DATA_ERROR: - PyErr_SetString(state->error_type, "Corrupt input data"); + PyErr_SetString(state->error, "Corrupt input data"); return 1; case LZMA_BUF_ERROR: - PyErr_SetString(state->error_type, "Insufficient buffer space"); + PyErr_SetString(state->error, "Insufficient buffer space"); return 1; case LZMA_PROG_ERROR: - PyErr_SetString(state->error_type, "Internal error"); + PyErr_SetString(state->error, "Internal error"); return 1; default: - PyErr_Format(state->error_type, "Unrecognized error from liblzma: %d", lzret); + PyErr_Format(state->error, "Unrecognized error from liblzma: %d", lzret); return 1; } } @@ -230,7 +230,7 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) if (lzma_lzma_preset(options, preset)) { PyMem_Free(options); - PyErr_Format(state->error_type, "Invalid compression preset: %u", preset); + PyErr_Format(state->error, "Invalid compression preset: %u", preset); return NULL; } @@ -648,7 +648,7 @@ Compressor_init_alone(_lzma_state *state, lzma_stream *lzs, uint32_t preset, PyO lzma_options_lzma options; if (lzma_lzma_preset(&options, preset)) { - PyErr_Format(state->error_type, "Invalid compression preset: %u", preset); + PyErr_Format(state->error, "Invalid compression preset: %u", preset); return -1; } lzret = lzma_alone_encoder(lzs, &options); @@ -1568,13 +1568,12 @@ lzma_exec(PyObject *module) ADD_INT_PREFIX_MACRO(module, PRESET_DEFAULT); ADD_INT_PREFIX_MACRO(module, PRESET_EXTREME); - - state->error_type = PyErr_NewExceptionWithDoc("_lzma.LZMAError", "Call to liblzma failed.", NULL, NULL); - if (state->error_type == NULL) { + state->error = PyErr_NewExceptionWithDoc("_lzma.LZMAError", "Call to liblzma failed.", NULL, NULL); + if (state->error == NULL) { return -1; } - if (PyModule_AddType(module, (PyTypeObject *)state->error_type) < 0) { + if (PyModule_AddType(module, (PyTypeObject *)state->error) < 0) { return -1; } @@ -1620,7 +1619,7 @@ lzma_traverse(PyObject *module, visitproc visit, void *arg) _lzma_state *state = get_lzma_state(module); Py_VISIT(state->lzma_compressor_type); Py_VISIT(state->lzma_decompressor_type); - Py_VISIT(state->error_type); + Py_VISIT(state->error); Py_VISIT(state->empty_tuple); return 0; } @@ -1631,7 +1630,7 @@ lzma_clear(PyObject *module) _lzma_state *state = get_lzma_state(module); Py_CLEAR(state->lzma_compressor_type); Py_CLEAR(state->lzma_decompressor_type); - Py_CLEAR(state->error_type); + Py_CLEAR(state->error); Py_CLEAR(state->empty_tuple); return 0; } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy