PostgreSQL Source Code  git master
toast_compression.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * toast_compression.c
4  * Functions for toast compression.
5  *
6  * Copyright (c) 2021-2024, PostgreSQL Global Development Group
7  *
8  *
9  * IDENTIFICATION
10  * src/backend/access/common/toast_compression.c
11  *
12  *-------------------------------------------------------------------------
13  */
14 #include "postgres.h"
15 
16 #ifdef USE_LZ4
17 #include <lz4.h>
18 #endif
19 
20 #include "access/detoast.h"
22 #include "common/pg_lzcompress.h"
23 #include "varatt.h"
24 
25 /* GUC */
27 
28 #define NO_LZ4_SUPPORT() \
29  ereport(ERROR, \
30  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
31  errmsg("compression method lz4 not supported"), \
32  errdetail("This functionality requires the server to be built with lz4 support.")))
33 
34 /*
35  * Compress a varlena using PGLZ.
36  *
37  * Returns the compressed varlena, or NULL if compression fails.
38  */
39 struct varlena *
41 {
42  int32 valsize,
43  len;
44  struct varlena *tmp = NULL;
45 
46  valsize = VARSIZE_ANY_EXHDR(value);
47 
48  /*
49  * No point in wasting a palloc cycle if value size is outside the allowed
50  * range for compression.
51  */
52  if (valsize < PGLZ_strategy_default->min_input_size ||
54  return NULL;
55 
56  /*
57  * Figure out the maximum possible size of the pglz output, add the bytes
58  * that will be needed for varlena overhead, and allocate that amount.
59  */
60  tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
62 
64  valsize,
65  (char *) tmp + VARHDRSZ_COMPRESSED,
66  NULL);
67  if (len < 0)
68  {
69  pfree(tmp);
70  return NULL;
71  }
72 
74 
75  return tmp;
76 }
77 
78 /*
79  * Decompress a varlena that was compressed using PGLZ.
80  */
81 struct varlena *
83 {
84  struct varlena *result;
85  int32 rawsize;
86 
87  /* allocate memory for the uncompressed data */
89 
90  /* decompress the data */
91  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
93  VARDATA(result),
95  if (rawsize < 0)
96  ereport(ERROR,
98  errmsg_internal("compressed pglz data is corrupt")));
99 
100  SET_VARSIZE(result, rawsize + VARHDRSZ);
101 
102  return result;
103 }
104 
105 /*
106  * Decompress part of a varlena that was compressed using PGLZ.
107  */
108 struct varlena *
110  int32 slicelength)
111 {
112  struct varlena *result;
113  int32 rawsize;
114 
115  /* allocate memory for the uncompressed data */
116  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
117 
118  /* decompress the data */
119  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
121  VARDATA(result),
122  slicelength, false);
123  if (rawsize < 0)
124  ereport(ERROR,
126  errmsg_internal("compressed pglz data is corrupt")));
127 
128  SET_VARSIZE(result, rawsize + VARHDRSZ);
129 
130  return result;
131 }
132 
133 /*
134  * Compress a varlena using LZ4.
135  *
136  * Returns the compressed varlena, or NULL if compression fails.
137  */
138 struct varlena *
140 {
141 #ifndef USE_LZ4
142  NO_LZ4_SUPPORT();
143  return NULL; /* keep compiler quiet */
144 #else
145  int32 valsize;
146  int32 len;
147  int32 max_size;
148  struct varlena *tmp = NULL;
149 
150  valsize = VARSIZE_ANY_EXHDR(value);
151 
152  /*
153  * Figure out the maximum possible size of the LZ4 output, add the bytes
154  * that will be needed for varlena overhead, and allocate that amount.
155  */
156  max_size = LZ4_compressBound(valsize);
157  tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
158 
159  len = LZ4_compress_default(VARDATA_ANY(value),
160  (char *) tmp + VARHDRSZ_COMPRESSED,
161  valsize, max_size);
162  if (len <= 0)
163  elog(ERROR, "lz4 compression failed");
164 
165  /* data is incompressible so just free the memory and return NULL */
166  if (len > valsize)
167  {
168  pfree(tmp);
169  return NULL;
170  }
171 
173 
174  return tmp;
175 #endif
176 }
177 
178 /*
179  * Decompress a varlena that was compressed using LZ4.
180  */
181 struct varlena *
183 {
184 #ifndef USE_LZ4
185  NO_LZ4_SUPPORT();
186  return NULL; /* keep compiler quiet */
187 #else
188  int32 rawsize;
189  struct varlena *result;
190 
191  /* allocate memory for the uncompressed data */
193 
194  /* decompress the data */
195  rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESSED,
196  VARDATA(result),
199  if (rawsize < 0)
200  ereport(ERROR,
202  errmsg_internal("compressed lz4 data is corrupt")));
203 
204 
205  SET_VARSIZE(result, rawsize + VARHDRSZ);
206 
207  return result;
208 #endif
209 }
210 
211 /*
212  * Decompress part of a varlena that was compressed using LZ4.
213  */
214 struct varlena *
215 lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
216 {
217 #ifndef USE_LZ4
218  NO_LZ4_SUPPORT();
219  return NULL; /* keep compiler quiet */
220 #else
221  int32 rawsize;
222  struct varlena *result;
223 
224  /* slice decompression not supported prior to 1.8.3 */
225  if (LZ4_versionNumber() < 10803)
226  return lz4_decompress_datum(value);
227 
228  /* allocate memory for the uncompressed data */
229  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
230 
231  /* decompress the data */
232  rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESSED,
233  VARDATA(result),
235  slicelength,
236  slicelength);
237  if (rawsize < 0)
238  ereport(ERROR,
240  errmsg_internal("compressed lz4 data is corrupt")));
241 
242  SET_VARSIZE(result, rawsize + VARHDRSZ);
243 
244  return result;
245 #endif
246 }
247 
248 /*
249  * Extract compression ID from a varlena.
250  *
251  * Returns TOAST_INVALID_COMPRESSION_ID if the varlena is not compressed.
252  */
255 {
257 
258  /*
259  * If it is stored externally then fetch the compression method id from
260  * the external toast pointer. If compressed inline, fetch it from the
261  * toast compression header.
262  */
263  if (VARATT_IS_EXTERNAL_ONDISK(attr))
264  {
265  struct varatt_external toast_pointer;
266 
267  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
268 
269  if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
270  cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
271  }
272  else if (VARATT_IS_COMPRESSED(attr))
274 
275  return cmid;
276 }
277 
278 /*
279  * CompressionNameToMethod - Get compression method from compression name
280  *
281  * Search in the available built-in methods. If the compression not found
282  * in the built-in methods then return InvalidCompressionMethod.
283  */
284 char
285 CompressionNameToMethod(const char *compression)
286 {
287  if (strcmp(compression, "pglz") == 0)
288  return TOAST_PGLZ_COMPRESSION;
289  else if (strcmp(compression, "lz4") == 0)
290  {
291 #ifndef USE_LZ4
292  NO_LZ4_SUPPORT();
293 #endif
294  return TOAST_LZ4_COMPRESSION;
295  }
296 
298 }
299 
300 /*
301  * GetCompressionMethodName - Get compression method name
302  */
303 const char *
305 {
306  switch (method)
307  {
309  return "pglz";
311  return "lz4";
312  default:
313  elog(ERROR, "invalid compression method %c", method);
314  return NULL; /* keep compiler quiet */
315  }
316 }
signed int int32
Definition: c.h:494
#define VARHDRSZ
Definition: c.h:692
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
int errmsg_internal(const char *fmt,...)
Definition: elog.c:1157
int errcode(int sqlerrcode)
Definition: elog.c:857
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:224
#define ereport(elevel,...)
Definition: elog.h:149
static struct @155 value
void pfree(void *pointer)
Definition: mcxt.c:1520
void * palloc(Size size)
Definition: mcxt.c:1316
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:41
const void size_t len
const PGLZ_Strategy *const PGLZ_strategy_default
int32 pglz_decompress(const char *source, int32 slen, char *dest, int32 rawsize, bool check_complete)
int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy)
#define PGLZ_MAX_OUTPUT(_dlen)
Definition: pg_lzcompress.h:21
int32 max_input_size
Definition: pg_lzcompress.h:60
Definition: c.h:687
struct varlena * lz4_decompress_datum(const struct varlena *value)
#define NO_LZ4_SUPPORT()
struct varlena * lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
int default_toast_compression
struct varlena * pglz_decompress_datum(const struct varlena *value)
struct varlena * pglz_decompress_datum_slice(const struct varlena *value, int32 slicelength)
const char * GetCompressionMethodName(char method)
struct varlena * lz4_compress_datum(const struct varlena *value)
struct varlena * pglz_compress_datum(const struct varlena *value)
char CompressionNameToMethod(const char *compression)
ToastCompressionId toast_get_compression_id(struct varlena *attr)
ToastCompressionId
@ TOAST_INVALID_COMPRESSION_ID
#define TOAST_PGLZ_COMPRESSION
#define InvalidCompressionMethod
#define TOAST_LZ4_COMPRESSION
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: varatt.h:290
#define SET_VARSIZE_COMPRESSED(PTR, len)
Definition: varatt.h:307
#define VARDATA(PTR)
Definition: varatt.h:278
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer)
Definition: varatt.h:336
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR)
Definition: varatt.h:328
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)
Definition: varatt.h:354
#define VARATT_IS_COMPRESSED(PTR)
Definition: varatt.h:288
#define VARDATA_ANY(PTR)
Definition: varatt.h:324
#define VARHDRSZ_COMPRESSED
Definition: varatt.h:254
#define SET_VARSIZE(PTR, len)
Definition: varatt.h:305
#define VARSIZE(PTR)
Definition: varatt.h:279
#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR)
Definition: varatt.h:330
#define VARSIZE_ANY_EXHDR(PTR)
Definition: varatt.h:317