Skip to content

Commit 19f6e9e

Browse files
alubennikovaalubennikova
committed
Fix typos, add comments, rename couple of variables
1 parent 410aa2d commit 19f6e9e

File tree

3 files changed

+134
-57
lines changed

3 files changed

+134
-57
lines changed

‎src/backend/storage/file/cfs.c

Lines changed: 118 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,16 @@
1111
*
1212
* NOTES:
1313
*
14-
* This file implemets compression of file pages.
14+
* This file implements compression of file pages.
1515
* Updated compressed pages are always appended to the end of file segment.
16-
* Garbage collector is used to shrink files when them become tool large.
17-
* GC is spawned as one or more background workers. Them recursively traverse all tablespace directories,
18-
* find out *.cfm files are if logical size of the file is twice larger than physical size of the file
19-
* performs compactification. Locking implemented using atomic operations is used to eliminate race
20-
* conditions.
16+
* Garbage collector is used to reclaim storage occupied by outdated versions of pages.
17+
* GC runs one or more background workers which recursively traverse all tablespace
18+
* directories. If worker finds out that logical size of the file is twice as large as
19+
* physical size of the file, it performs compactification.
20+
* To eliminate race conditions, files are locked during compacification.
21+
* Locks are implemented with atomic operations.
22+
*
23+
* TODO Write a note about *.cfm files.
2124
*/
2225

2326
#include "postgres.h"
@@ -51,13 +54,37 @@
5154
#include "utils/resowner_private.h"
5255
#include "postmaster/bgworker.h"
5356

57+
/*
58+
* GUC variable that defines compression level.
59+
* 0 - no compression, 1 - max speed,
60+
* other possible values depend on the specific algorithm.
61+
* Default value is 1.
62+
*/
63+
int cfs_level;
64+
/*
65+
* GUC variable that defines if encryption of compressed pages is enabled.
66+
* Default value is false.
67+
*/
68+
bool cfs_encryption;
69+
/*
70+
* GUC variable - Verify correctness of data written by GC.
71+
* TODO add description and documentation.
72+
*/
73+
bool cfs_gc_verify_file;
74+
75+
/* GUC variable - Number of garbage collection background workers. Default = 1 */
5476
int cfs_gc_workers;
77+
/*
78+
* GUC variable - Specifies the minimum percent of garbage blocks
79+
* needed to trigger a GC of the file. Default = 50
80+
*/
5581
int cfs_gc_threshold;
82+
/* GUC variable - Time to sleep between GC runs in milliseconds. Default = 5000 */
5683
int cfs_gc_period;
84+
/* GUC variable - Delay in milliseconds between files defragmentation. Default = 0
85+
* TODO What is the purpose of this variable?
86+
*/
5787
int cfs_gc_delay;
58-
int cfs_level;
59-
bool cfs_encryption;
60-
bool cfs_gc_verify_file;
6188

6289
static bool cfs_read_file(int fd, void* data, uint32 size);
6390
static bool cfs_write_file(int fd, void const* data, uint32 size);
@@ -68,90 +95,109 @@ CfsState* cfs_state;
6895
static bool cfs_stop;
6996
static int cfs_processed_segments;
7097

71-
#if CFS_COMPRESSOR == SNAPPY_COMPRESSOR
7298

73-
#include <snappy-c.h>
99+
/* ----------------------------------------------------------------
100+
* Section 1: Various compression algorithms.
101+
* CFS_COMPRESSOR variable can be set at compile time.
102+
* One should define CFS_COMPRESSOR in cfs.h
103+
* Availiable options are:
104+
* - LZ_COMPRESSOR // FIXME. No actual implementation.
105+
* - ZLIB_COMPRESSOR
106+
* - LZ4_COMPRESSOR
107+
* - SNAPPY_COMPRESSOR
108+
* - LCFSE_COMPRESSOR
109+
* - ZSTD_COMPRESSOR
110+
*
111+
* If none of options is chosen, use standard pglz_compress FIXME?, which is
112+
* slow and non-efficient in comparison with others, but doesn't requre
113+
* any extra libraries.
114+
* ----------------------------------------------------------------
115+
*/
116+
117+
#if CFS_COMPRESSOR == ZLIB_COMPRESSOR
118+
119+
#include <zlib.h>
74120

75121
size_t cfs_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
76122
{
77-
return snappy_compress(src, src_size, dst, &dst_size) == SNAPPY_OK ? dst_size : 0;
123+
uLongf compressed_size = dst_size;
124+
int rc = compress2(dst, &compressed_size, src, src_size, cfs_level);
125+
return rc == Z_OK ? compressed_size : rc;
78126
}
79127

80128
size_t cfs_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
81129
{
82-
return snappy_uncompress(src, src_size, dst, &dst_size) == SNAPPY_OK ? dst_size : 0;
130+
uLongf dest_len = dst_size;
131+
int rc = uncompress(dst, &dest_len, src, src_size);
132+
return rc == Z_OK ? dest_len : rc;
83133
}
84134

85135
char const* cfs_algorithm()
86136
{
87-
return "snappy";
137+
return "zlib";
88138
}
89139

90-
#elif CFS_COMPRESSOR == LCFSE_COMPRESSOR
140+
#elif CFS_COMPRESSOR == LZ4_COMPRESSOR
91141

92-
#include <lcfse.h>
142+
#include <lz4.h>
93143

94144
size_t cfs_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
95145
{
96-
char* scratch_buf = palloc(lcfse_encode_scratch_size());
97-
size_t rc = lcfse_encode_buffer(dst, dst_size, src, src_size, scratch_buf);
98-
pfree(scratch_buf);
99-
return rc;
146+
return LZ4_compress(src, dst, src_size);
100147
}
101148

102149
size_t cfs_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
103150
{
104-
char* scratch_buf = palloc(lcfse_encode_scratch_size());
105-
size_t rc = lcfse_decode_buffer(dst, dst_size, src, src_size, scratch_buf);
106-
pfree(scratch_buf);
107-
return rc;
151+
return LZ4_decompress_safe(src, dst, src_size, dst_size);
108152
}
109153

110154
char const* cfs_algorithm()
111155
{
112-
return "lcfse";
156+
return "lz4";
113157
}
114158

115-
#elif CFS_COMPRESSOR == LZ4_COMPRESSOR
159+
#elif CFS_COMPRESSOR == SNAPPY_COMPRESSOR
116160

117-
#include <lz4.h>
161+
#include <snappy-c.h>
118162

119163
size_t cfs_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
120164
{
121-
return LZ4_compress(src, dst, src_size);
165+
return snappy_compress(src, src_size, dst, &dst_size) == SNAPPY_OK ? dst_size : 0;
122166
}
123167

124168
size_t cfs_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
125169
{
126-
return LZ4_decompress_safe(src, dst, src_size, dst_size);
170+
return snappy_uncompress(src, src_size, dst, &dst_size) == SNAPPY_OK ? dst_size : 0;
127171
}
128172

129173
char const* cfs_algorithm()
130174
{
131-
return "lz4";
175+
return "snappy";
132176
}
133177

134-
#elif CFS_COMPRESSOR == ZLIB_COMPRESSOR
178+
#elif CFS_COMPRESSOR == LCFSE_COMPRESSOR
135179

136-
#include <zlib.h>
180+
#include <lcfse.h>
137181

138182
size_t cfs_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
139183
{
140-
uLongf compressed_size = dst_size;
141-
int rc = compress2(dst, &compressed_size, src, src_size, cfs_level);
142-
return rc == Z_OK ? compressed_size : rc;
184+
char* scratch_buf = palloc(lcfse_encode_scratch_size());
185+
size_t rc = lcfse_encode_buffer(dst, dst_size, src, src_size, scratch_buf);
186+
pfree(scratch_buf);
187+
return rc;
143188
}
144189

145190
size_t cfs_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
146191
{
147-
uLongf dest_len = dst_size;
148-
int rc = uncompress(dst, &dest_len, src, src_size);
149-
return rc == Z_OK ? dest_len : rc;
192+
char* scratch_buf = palloc(lcfse_encode_scratch_size());
193+
size_t rc = lcfse_decode_buffer(dst, dst_size, src, src_size, scratch_buf);
194+
pfree(scratch_buf);
195+
return rc;
150196
}
151197

152198
char const* cfs_algorithm()
153199
{
154-
return "zlib";
200+
return "lcfse";
155201
}
156202

157203
#elif CFS_COMPRESSOR == ZSTD_COMPRESSOR
@@ -195,6 +241,15 @@ char const* cfs_algorithm()
195241
#endif
196242

197243

244+
/* ----------------------------------------------------------------
245+
* Section 2: Encryption related functionality.
246+
*
247+
* TODO
248+
* - replace rc4 algrithm with something more appropriate
249+
* - add more comments
250+
* - what does 'offs' variable for?
251+
* ----------------------------------------------------------------
252+
*/
198253
static void cfs_rc4_encrypt_block(void* block, uint32 offs, uint32 block_size)
199254
{
200255
uint32 i;
@@ -205,7 +260,7 @@ static void cfs_rc4_encrypt_block(void* block, uint32 offs, uint32 block_size)
205260
int x = 0, y = 0;
206261
uint32 skip = (offs / BLCKSZ + block_size) % CFS_CIPHER_KEY_SIZE;
207262

208-
memcpy(state, cfs_state->rc4_init_state, CFS_CIPHER_KEY_SIZE);
263+
memcpy(state, cfs_state->cipher_key, CFS_CIPHER_KEY_SIZE);
209264
for (i = 0; i < skip; i++) {
210265
x = (x + 1) % CFS_CIPHER_KEY_SIZE;
211266
y = (y + state[x]) % CFS_CIPHER_KEY_SIZE;
@@ -224,7 +279,12 @@ static void cfs_rc4_encrypt_block(void* block, uint32 offs, uint32 block_size)
224279
}
225280
}
226281

227-
static void cfs_rc4_init(void)
282+
/*
283+
* Get env variable PG_CIPHER_KEY and initialize encryption state.
284+
* Unset variable afterward.
285+
* Now implements cf4.
286+
*/
287+
static void cfs_encrypt_init(void)
228288
{
229289
int index1 = 0;
230290
int index2 = 0;
@@ -233,13 +293,13 @@ static void cfs_rc4_init(void)
233293
int key_length;
234294
int x = 0, y = 0;
235295
char* cipher_key;
236-
uint8* rc4_init_state = cfs_state->rc4_init_state;
296+
uint8* rc4_init_state = cfs_state->cipher_key;
237297

238298
cipher_key = getenv("PG_CIPHER_KEY");
239299
if (cipher_key == NULL) {
240300
elog(ERROR, "PG_CIPHER_KEY environment variable is not set");
241301
}
242-
unsetenv("PG_CIPHER_KEY"); /* make it not possible to inspect this environment variable through plperl */
302+
unsetenv("PG_CIPHER_KEY"); /* disable inspection of this environment variable */
243303
key_length = strlen(cipher_key);
244304
for (i = 0; i < CFS_CIPHER_KEY_SIZE; ++i) {
245305
rc4_init_state[i] = (uint8)i;
@@ -263,20 +323,21 @@ static void cfs_rc4_init(void)
263323
void cfs_encrypt(void* block, uint32 offs, uint32 size)
264324
{
265325
if (cfs_encryption)
266-
{
267326
cfs_rc4_encrypt_block(block, offs, size);
268-
}
269327
}
270328

271329
void cfs_decrypt(void* block, uint32 offs, uint32 size)
272330
{
273331
if (cfs_encryption)
274-
{
275332
cfs_rc4_encrypt_block(block, offs, size);
276-
}
277333
}
278334

279-
335+
/* ----------------------------------------------------------------
336+
* Section 3: Compression implementation.
337+
*
338+
* TODO add description
339+
* ----------------------------------------------------------------
340+
*/
280341
void cfs_initialize()
281342
{
282343
cfs_state = (CfsState*)ShmemAlloc(sizeof(CfsState));
@@ -287,9 +348,9 @@ void cfs_initialize()
287348
cfs_state->gc_enabled = true;
288349
cfs_state->max_iterations = 0;
289350

290-
if (cfs_encryption) {
291-
cfs_rc4_init();
292-
}
351+
if (cfs_encryption)
352+
cfs_encrypt_init();
353+
293354
elog(LOG, "Start CFS version %s compression algorithm %s encryption %s",
294355
CFS_VERSION, cfs_algorithm(), cfs_encryption ? "enabled" : "disabled");
295356
}
@@ -463,6 +524,12 @@ static int cfs_cmp_page_offs(void const* p1, void const* p2)
463524
return o1 < o2 ? -1 : o1 == o2 ? 0 : 1;
464525
}
465526

527+
/* ----------------------------------------------------------------
528+
* Section 4: Garbage collection functionality.
529+
*
530+
* TODO add description. reorder functions.
531+
* ----------------------------------------------------------------
532+
*/
466533
/*
467534
* Perform garbage collection (if required) of file
468535
* @param map_path path to file map file (*.cfm).

‎src/backend/utils/misc/guc.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2825,7 +2825,7 @@ static struct config_int ConfigureNamesInt[] =
28252825

28262826
{
28272827
{"cfs_gc_threshold", PGC_USERSET, UNGROUPED,
2828-
gettext_noop("Percent of garbage in file after file is comactified"),
2828+
gettext_noop("Minimum percent of garbage blocks in the file prior to garbage collection"),
28292829
NULL,
28302830
0
28312831
},
@@ -2837,7 +2837,7 @@ static struct config_int ConfigureNamesInt[] =
28372837

28382838
{
28392839
{"cfs_gc_period", PGC_USERSET, UNGROUPED,
2840-
gettext_noop("Interval in milliseconds between CFS garbage collection iterations"),
2840+
gettext_noop("Time to sleep between GC runs in milliseconds"),
28412841
NULL,
28422842
GUC_UNIT_MS
28432843
},

‎src/include/storage/cfs.h

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,25 @@
1717
#define CFS_MAX_COMPRESSED_SIZE(size) ((size)*2)
1818
#define CFS_MIN_COMPRESSED_SIZE(size) ((size)*2/3)
1919

20+
/*
21+
* FIXME
22+
*/
2023
#define LZ_COMPRESSOR 1
2124
#define ZLIB_COMPRESSOR 2
2225
#define LZ4_COMPRESSOR 3
2326
#define SNAPPY_COMPRESSOR 4
2427
#define LCFSE_COMPRESSOR 5
2528
#define ZSTD_COMPRESSOR 6
2629

30+
/*
31+
* Set CFS_COMPRESSOR to one of the names above
32+
* to compile postgres with chosen compression algorithm
33+
*/
2734
#ifndef CFS_COMPRESSOR
2835
#define CFS_COMPRESSOR ZLIB_COMPRESSOR
2936
#endif
3037

38+
/* Encryption related variables*/
3139
#define CFS_RC4_DROP_N 3072
3240
#define CFS_CIPHER_KEY_SIZE 256
3341

@@ -54,6 +62,7 @@ typedef struct
5462
uint64 processedBytes;
5563
} CfsStatistic;
5664

65+
/* TODO Add comments */
5766
typedef struct
5867
{
5968
pg_atomic_flag gc_started;
@@ -62,7 +71,7 @@ typedef struct
6271
int max_iterations;
6372
bool gc_enabled;
6473
CfsStatistic gc_stat;
65-
uint8 rc4_init_state[CFS_CIPHER_KEY_SIZE];
74+
uint8 cipher_key[CFS_CIPHER_KEY_SIZE]; /* FIXME Is it the key itself? Or some modification?*/
6675
} CfsState;
6776

6877
typedef struct
@@ -91,13 +100,14 @@ void cfs_decrypt(void* block, uint32 offs, uint32 size);
91100

92101
extern CfsState* cfs_state;
93102

103+
extern int cfs_level;
104+
extern bool cfs_encryption;
105+
extern bool cfs_gc_verify_file;
106+
94107
extern int cfs_gc_delay;
95108
extern int cfs_gc_period;
96109
extern int cfs_gc_workers;
97110
extern int cfs_gc_threshold;
98-
extern int cfs_level;
99-
extern bool cfs_gc_verify_file;
100-
extern bool cfs_encryption;
101111
#endif
102112

103113

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy