Skip to content

Commit f7c441c

Browse files
miss-islingtonAA-Turner
miss-islington
and
AA-Turner
authored
[3.14] gh-132983: Reduce the size of _zstdmodule.h (GH-133793) (#133854)
gh-132983: Reduce the size of ``_zstdmodule.h`` (GH-133793) (cherry picked from commit 1a548c0) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>
1 parent 39485d5 commit f7c441c

File tree

10 files changed

+165
-164
lines changed

10 files changed

+165
-164
lines changed

‎Makefile.pre.in

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3346,7 +3346,7 @@ MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_tes
33463346
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
33473347
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h
33483348
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
3349-
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h
3349+
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
33503350

33513351
CODECS_COMMON_HEADERS=$(srcdir)/Modules/cjkcodecs/multibytecodec.h $(srcdir)/Modules/cjkcodecs/cjkcodecs.h
33523352
MODULE__CODECS_CN_DEPS=$(srcdir)/Modules/cjkcodecs/mappings_cn.h $(CODECS_COMMON_HEADERS)

‎Modules/_zstd/_zstdmodule.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,13 @@ Python module.
77
# define Py_BUILD_CORE_MODULE 1
88
#endif
99

10+
#include "Python.h"
11+
1012
#include "_zstdmodule.h"
13+
#include "zstddict.h"
14+
15+
#include <zstd.h> // ZSTD_*()
16+
#include <zdict.h> // ZDICT_*()
1117

1218
/*[clinic input]
1319
module _zstd
@@ -727,7 +733,7 @@ static struct PyModuleDef_Slot _zstd_slots[] = {
727733
{0, NULL},
728734
};
729735

730-
struct PyModuleDef _zstdmodule = {
736+
static struct PyModuleDef _zstdmodule = {
731737
.m_base = PyModuleDef_HEAD_INIT,
732738
.m_name = "_zstd",
733739
.m_doc = "Implementation module for Zstandard compression.",

‎Modules/_zstd/_zstdmodule.h

Lines changed: 8 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -1,132 +1,29 @@
1-
#pragma once
21
/*
32
Low level interface to Meta's zstd library for use in the compression.zstd
43
Python module.
54
*/
65

76
/* Declarations shared between different parts of the _zstd module*/
87

9-
#include "Python.h"
10-
11-
#include "zstd.h"
12-
#include "zdict.h"
13-
14-
15-
/* Forward declaration of module state */
16-
typedef struct _zstd_state _zstd_state;
17-
18-
/* Forward reference of module def */
19-
extern PyModuleDef _zstdmodule;
20-
21-
/* For clinic type calculations */
22-
static inline _zstd_state *
23-
get_zstd_state_from_type(PyTypeObject *type)
24-
{
25-
PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule);
26-
if (module == NULL) {
27-
return NULL;
28-
}
29-
void *state = PyModule_GetState(module);
30-
assert(state != NULL);
31-
return (_zstd_state *)state;
32-
}
8+
#ifndef ZSTD_MODULE_H
9+
#define ZSTD_MODULE_H
3310

11+
/* Type specs */
3412
extern PyType_Spec zstd_dict_type_spec;
3513
extern PyType_Spec zstd_compressor_type_spec;
3614
extern PyType_Spec zstd_decompressor_type_spec;
3715

38-
struct _zstd_state {
16+
typedef struct {
17+
/* Module heap types. */
3918
PyTypeObject *ZstdDict_type;
4019
PyTypeObject *ZstdCompressor_type;
4120
PyTypeObject *ZstdDecompressor_type;
4221
PyObject *ZstdError;
4322

23+
/* enum types set by set_parameter_types. */
4424
PyTypeObject *CParameter_type;
4525
PyTypeObject *DParameter_type;
46-
};
47-
48-
typedef struct {
49-
PyObject_HEAD
50-
51-
/* Reusable compress/decompress dictionary, they are created once and
52-
can be shared by multiple threads concurrently, since its usage is
53-
read-only.
54-
c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */
55-
ZSTD_DDict *d_dict;
56-
PyObject *c_dicts;
57-
58-
/* Content of the dictionary, bytes object. */
59-
PyObject *dict_content;
60-
/* Dictionary id */
61-
uint32_t dict_id;
62-
63-
/* __init__ has been called, 0 or 1. */
64-
int inited;
65-
} ZstdDict;
66-
67-
typedef struct {
68-
PyObject_HEAD
69-
70-
/* Compression context */
71-
ZSTD_CCtx *cctx;
72-
73-
/* ZstdDict object in use */
74-
PyObject *dict;
75-
76-
/* Last mode, initialized to ZSTD_e_end */
77-
int last_mode;
78-
79-
/* (nbWorker >= 1) ? 1 : 0 */
80-
int use_multithread;
81-
82-
/* Compression level */
83-
int compression_level;
84-
85-
/* __init__ has been called, 0 or 1. */
86-
int inited;
87-
} ZstdCompressor;
88-
89-
typedef struct {
90-
PyObject_HEAD
91-
92-
/* Decompression context */
93-
ZSTD_DCtx *dctx;
94-
95-
/* ZstdDict object in use */
96-
PyObject *dict;
97-
98-
/* Unconsumed input data */
99-
char *input_buffer;
100-
size_t input_buffer_size;
101-
size_t in_begin, in_end;
102-
103-
/* Unused data */
104-
PyObject *unused_data;
105-
106-
/* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */
107-
char needs_input;
108-
109-
/* For decompress(), 0 or 1.
110-
1 when both input and output streams are at a frame edge, means a
111-
frame is completely decoded and fully flushed, or the decompressor
112-
just be initialized. */
113-
char at_frame_edge;
114-
115-
/* For ZstdDecompressor, 0 or 1.
116-
1 means the end of the first frame has been reached. */
117-
char eof;
118-
119-
/* Used for fast reset above three variables */
120-
char _unused_char_for_align;
121-
122-
/* __init__ has been called, 0 or 1. */
123-
int inited;
124-
} ZstdDecompressor;
125-
126-
typedef enum {
127-
TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class
128-
TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function
129-
} decompress_type;
26+
} _zstd_state;
13027

13128
typedef enum {
13229
ERR_DECOMPRESS,
@@ -149,12 +46,6 @@ typedef enum {
14946
DICT_TYPE_PREFIX = 2
15047
} dictionary_type;
15148

152-
static inline int
153-
mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
154-
{
155-
return in->size == in->pos && out->size != out->pos;
156-
}
157-
15849
/* Format error message and set ZstdError. */
15950
extern void
16051
set_zstd_error(const _zstd_state* const state,
@@ -164,4 +55,4 @@ extern void
16455
set_parameter_error(const _zstd_state* const state, int is_compress,
16556
int key_v, int value_v);
16657

167-
static const char init_twice_msg[] = "__init__ method is called twice.";
58+
#endif // !ZSTD_MODULE_H

‎Modules/_zstd/buffer.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,13 @@ Low level interface to Meta's zstd library for use in the compression.zstd
33
Python module.
44
*/
55

6-
#include "_zstdmodule.h"
6+
#ifndef ZSTD_BUFFER_H
7+
#define ZSTD_BUFFER_H
8+
79
#include "pycore_blocks_output_buffer.h"
810

11+
#include <zstd.h> // ZSTD_outBuffer
12+
913
/* Blocks output buffer wrapper code */
1014

1115
/* Initialize the buffer, and grow the buffer.
@@ -102,3 +106,5 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob)
102106

103107
return buffer->allocated == buffer->max_length;
104108
}
109+
110+
#endif // !ZSTD_BUFFER_H

‎Modules/_zstd/compressor.c

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,50 @@ Python module.
77

88
/*[clinic input]
99
module _zstd
10-
class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_type"
10+
class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec"
1111
[clinic start generated code]*/
12-
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/
13-
12+
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7166021db1ef7df8]*/
1413

1514
#ifndef Py_BUILD_CORE_BUILTIN
1615
# define Py_BUILD_CORE_MODULE 1
1716
#endif
1817

19-
#include "_zstdmodule.h"
18+
#include "Python.h"
2019

20+
#include "_zstdmodule.h"
2121
#include "buffer.h"
22+
#include "zstddict.h"
2223

24+
#include <stdbool.h> // bool
2325
#include <stddef.h> // offsetof()
26+
#include <zstd.h> // ZSTD_*()
27+
28+
typedef struct {
29+
PyObject_HEAD
30+
31+
/* Compression context */
32+
ZSTD_CCtx *cctx;
33+
34+
/* ZstdDict object in use */
35+
PyObject *dict;
36+
37+
/* Last mode, initialized to ZSTD_e_end */
38+
int last_mode;
39+
40+
/* (nbWorker >= 1) ? 1 : 0 */
41+
int use_multithread;
2442

43+
/* Compression level */
44+
int compression_level;
45+
46+
/* __init__ has been called, 0 or 1. */
47+
bool initialized;
48+
} ZstdCompressor;
2549

2650
#define ZstdCompressor_CAST(op) ((ZstdCompressor *)op)
2751

52+
#include "clinic/compressor.c.h"
53+
2854
static int
2955
_zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options,
3056
const char *arg_name, const char* arg_type)
@@ -292,10 +318,6 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict)
292318
return 0;
293319
}
294320

295-
#define clinic_state() (get_zstd_state_from_type(type))
296-
#include "clinic/compressor.c.h"
297-
#undef clinic_state
298-
299321
static PyObject *
300322
_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs))
301323
{
@@ -305,7 +327,7 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject
305327
goto error;
306328
}
307329

308-
self->inited = 0;
330+
self->initialized = 0;
309331
self->dict = NULL;
310332
self->use_multithread = 0;
311333

@@ -372,12 +394,11 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level,
372394
PyObject *options, PyObject *zstd_dict)
373395
/*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/
374396
{
375-
/* Only called once */
376-
if (self->inited) {
377-
PyErr_SetString(PyExc_RuntimeError, init_twice_msg);
397+
if (self->initialized) {
398+
PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported");
378399
return -1;
379400
}
380-
self->inited = 1;
401+
self->initialized = 1;
381402

382403
if (level != Py_None && options != Py_None) {
383404
PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used.");
@@ -488,6 +509,12 @@ compress_impl(ZstdCompressor *self, Py_buffer *data,
488509
return NULL;
489510
}
490511

512+
static inline int
513+
mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out)
514+
{
515+
return in->size == in->pos && out->size != out->pos;
516+
}
517+
491518
static PyObject *
492519
compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data)
493520
{

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy