PostgreSQL Source Code  git master
toast_compression.c File Reference
#include "postgres.h"
#include "access/detoast.h"
#include "access/toast_compression.h"
#include "common/pg_lzcompress.h"
#include "fmgr.h"
#include "utils/builtins.h"
#include "varatt.h"
Include dependency graph for toast_compression.c:

Go to the source code of this file.

Macros

#define NO_LZ4_SUPPORT()
 

Functions

struct varlenapglz_compress_datum (const struct varlena *value)
 
struct varlenapglz_decompress_datum (const struct varlena *value)
 
struct varlenapglz_decompress_datum_slice (const struct varlena *value, int32 slicelength)
 
struct varlenalz4_compress_datum (const struct varlena *value)
 
struct varlenalz4_decompress_datum (const struct varlena *value)
 
struct varlenalz4_decompress_datum_slice (const struct varlena *value, int32 slicelength)
 
ToastCompressionId toast_get_compression_id (struct varlena *attr)
 
char CompressionNameToMethod (const char *compression)
 
const char * GetCompressionMethodName (char method)
 

Variables

int default_toast_compression = TOAST_PGLZ_COMPRESSION
 

Macro Definition Documentation

◆ NO_LZ4_SUPPORT

#define NO_LZ4_SUPPORT ( )
Value:
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
errmsg("compression method lz4 not supported"), \
errdetail("This functionality requires the server to be built with lz4 support.")))
int errdetail(const char *fmt,...)
Definition: elog.c:1202
int errcode(int sqlerrcode)
Definition: elog.c:858
int errmsg(const char *fmt,...)
Definition: elog.c:1069
#define ERROR
Definition: elog.h:39
#define ereport(elevel,...)
Definition: elog.h:149

Definition at line 30 of file toast_compression.c.

Function Documentation

◆ CompressionNameToMethod()

char CompressionNameToMethod ( const char *  compression)

Definition at line 287 of file toast_compression.c.

288 {
289  if (strcmp(compression, "pglz") == 0)
290  return TOAST_PGLZ_COMPRESSION;
291  else if (strcmp(compression, "lz4") == 0)
292  {
293 #ifndef USE_LZ4
294  NO_LZ4_SUPPORT();
295 #endif
296  return TOAST_LZ4_COMPRESSION;
297  }
298 
300 }
#define NO_LZ4_SUPPORT()
#define TOAST_PGLZ_COMPRESSION
#define InvalidCompressionMethod
#define TOAST_LZ4_COMPRESSION

References InvalidCompressionMethod, NO_LZ4_SUPPORT, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by GetAttributeCompression().

◆ GetCompressionMethodName()

const char* GetCompressionMethodName ( char  method)

Definition at line 306 of file toast_compression.c.

307 {
308  switch (method)
309  {
311  return "pglz";
313  return "lz4";
314  default:
315  elog(ERROR, "invalid compression method %c", method);
316  return NULL; /* keep compiler quiet */
317  }
318 }

References elog(), ERROR, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by MergeAttributes(), and transformTableLikeClause().

◆ lz4_compress_datum()

struct varlena* lz4_compress_datum ( const struct varlena value)

Definition at line 141 of file toast_compression.c.

142 {
143 #ifndef USE_LZ4
144  NO_LZ4_SUPPORT();
145  return NULL; /* keep compiler quiet */
146 #else
147  int32 valsize;
148  int32 len;
149  int32 max_size;
150  struct varlena *tmp = NULL;
151 
152  valsize = VARSIZE_ANY_EXHDR(value);
153 
154  /*
155  * Figure out the maximum possible size of the LZ4 output, add the bytes
156  * that will be needed for varlena overhead, and allocate that amount.
157  */
158  max_size = LZ4_compressBound(valsize);
159  tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
160 
161  len = LZ4_compress_default(VARDATA_ANY(value),
162  (char *) tmp + VARHDRSZ_COMPRESSED,
163  valsize, max_size);
164  if (len <= 0)
165  elog(ERROR, "lz4 compression failed");
166 
167  /* data is incompressible so just free the memory and return NULL */
168  if (len > valsize)
169  {
170  pfree(tmp);
171  return NULL;
172  }
173 
175 
176  return tmp;
177 #endif
178 }
signed int int32
Definition: c.h:478
static struct @147 value
void pfree(void *pointer)
Definition: mcxt.c:1456
void * palloc(Size size)
Definition: mcxt.c:1226
const void size_t len
Definition: c.h:671
#define SET_VARSIZE_COMPRESSED(PTR, len)
Definition: varatt.h:307
#define VARDATA_ANY(PTR)
Definition: varatt.h:324
#define VARHDRSZ_COMPRESSED
Definition: varatt.h:254
#define VARSIZE_ANY_EXHDR(PTR)
Definition: varatt.h:317

References elog(), ERROR, len, NO_LZ4_SUPPORT, palloc(), pfree(), SET_VARSIZE_COMPRESSED, value, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by toast_compress_datum().

◆ lz4_decompress_datum()

struct varlena* lz4_decompress_datum ( const struct varlena value)

Definition at line 184 of file toast_compression.c.

185 {
186 #ifndef USE_LZ4
187  NO_LZ4_SUPPORT();
188  return NULL; /* keep compiler quiet */
189 #else
190  int32 rawsize;
191  struct varlena *result;
192 
193  /* allocate memory for the uncompressed data */
195 
196  /* decompress the data */
197  rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESSED,
198  VARDATA(result),
201  if (rawsize < 0)
202  ereport(ERROR,
204  errmsg_internal("compressed lz4 data is corrupt")));
205 
206 
207  SET_VARSIZE(result, rawsize + VARHDRSZ);
208 
209  return result;
210 #endif
211 }
#define VARHDRSZ
Definition: c.h:676
int errmsg_internal(const char *fmt,...)
Definition: elog.c:1156
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:41
#define VARDATA(PTR)
Definition: varatt.h:278
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR)
Definition: varatt.h:328
#define SET_VARSIZE(PTR, len)
Definition: varatt.h:305
#define VARSIZE(PTR)
Definition: varatt.h:279

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, value, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by lz4_decompress_datum_slice(), and toast_decompress_datum().

◆ lz4_decompress_datum_slice()

struct varlena* lz4_decompress_datum_slice ( const struct varlena value,
int32  slicelength 
)

Definition at line 217 of file toast_compression.c.

218 {
219 #ifndef USE_LZ4
220  NO_LZ4_SUPPORT();
221  return NULL; /* keep compiler quiet */
222 #else
223  int32 rawsize;
224  struct varlena *result;
225 
226  /* slice decompression not supported prior to 1.8.3 */
227  if (LZ4_versionNumber() < 10803)
228  return lz4_decompress_datum(value);
229 
230  /* allocate memory for the uncompressed data */
231  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
232 
233  /* decompress the data */
234  rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESSED,
235  VARDATA(result),
237  slicelength,
238  slicelength);
239  if (rawsize < 0)
240  ereport(ERROR,
242  errmsg_internal("compressed lz4 data is corrupt")));
243 
244  SET_VARSIZE(result, rawsize + VARHDRSZ);
245 
246  return result;
247 #endif
248 }
struct varlena * lz4_decompress_datum(const struct varlena *value)

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, lz4_decompress_datum(), NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, value, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum_slice().

◆ pglz_compress_datum()

struct varlena* pglz_compress_datum ( const struct varlena value)

Definition at line 42 of file toast_compression.c.

43 {
44  int32 valsize,
45  len;
46  struct varlena *tmp = NULL;
47 
48  valsize = VARSIZE_ANY_EXHDR(value);
49 
50  /*
51  * No point in wasting a palloc cycle if value size is outside the allowed
52  * range for compression.
53  */
54  if (valsize < PGLZ_strategy_default->min_input_size ||
56  return NULL;
57 
58  /*
59  * Figure out the maximum possible size of the pglz output, add the bytes
60  * that will be needed for varlena overhead, and allocate that amount.
61  */
62  tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
64 
66  valsize,
67  (char *) tmp + VARHDRSZ_COMPRESSED,
68  NULL);
69  if (len < 0)
70  {
71  pfree(tmp);
72  return NULL;
73  }
74 
76 
77  return tmp;
78 }
const PGLZ_Strategy *const PGLZ_strategy_default
int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy)
#define PGLZ_MAX_OUTPUT(_dlen)
Definition: pg_lzcompress.h:21
int32 max_input_size
Definition: pg_lzcompress.h:60

References len, PGLZ_Strategy::max_input_size, palloc(), pfree(), pglz_compress(), PGLZ_MAX_OUTPUT, PGLZ_strategy_default, SET_VARSIZE_COMPRESSED, value, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by toast_compress_datum().

◆ pglz_decompress_datum()

struct varlena* pglz_decompress_datum ( const struct varlena value)

Definition at line 84 of file toast_compression.c.

85 {
86  struct varlena *result;
87  int32 rawsize;
88 
89  /* allocate memory for the uncompressed data */
91 
92  /* decompress the data */
93  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
95  VARDATA(result),
97  if (rawsize < 0)
98  ereport(ERROR,
100  errmsg_internal("compressed pglz data is corrupt")));
101 
102  SET_VARSIZE(result, rawsize + VARHDRSZ);
103 
104  return result;
105 }
int32 pglz_decompress(const char *source, int32 slen, char *dest, int32 rawsize, bool check_complete)

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, value, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum().

◆ pglz_decompress_datum_slice()

struct varlena* pglz_decompress_datum_slice ( const struct varlena value,
int32  slicelength 
)

Definition at line 111 of file toast_compression.c.

113 {
114  struct varlena *result;
115  int32 rawsize;
116 
117  /* allocate memory for the uncompressed data */
118  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
119 
120  /* decompress the data */
121  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
123  VARDATA(result),
124  slicelength, false);
125  if (rawsize < 0)
126  ereport(ERROR,
128  errmsg_internal("compressed pglz data is corrupt")));
129 
130  SET_VARSIZE(result, rawsize + VARHDRSZ);
131 
132  return result;
133 }

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, value, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum_slice().

◆ toast_get_compression_id()

ToastCompressionId toast_get_compression_id ( struct varlena attr)

Definition at line 256 of file toast_compression.c.

257 {
259 
260  /*
261  * If it is stored externally then fetch the compression method id from
262  * the external toast pointer. If compressed inline, fetch it from the
263  * toast compression header.
264  */
265  if (VARATT_IS_EXTERNAL_ONDISK(attr))
266  {
267  struct varatt_external toast_pointer;
268 
269  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
270 
271  if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
272  cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
273  }
274  else if (VARATT_IS_COMPRESSED(attr))
276 
277  return cmid;
278 }
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
ToastCompressionId
@ TOAST_INVALID_COMPRESSION_ID
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: varatt.h:290
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer)
Definition: varatt.h:336
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)
Definition: varatt.h:354
#define VARATT_IS_COMPRESSED(PTR)
Definition: varatt.h:288
#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR)
Definition: varatt.h:330

References TOAST_INVALID_COMPRESSION_ID, VARATT_EXTERNAL_GET_COMPRESS_METHOD, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL_ONDISK, and VARDATA_COMPRESSED_GET_COMPRESS_METHOD.

Referenced by pg_column_compression().

Variable Documentation

◆ default_toast_compression

int default_toast_compression = TOAST_PGLZ_COMPRESSION

Definition at line 28 of file toast_compression.c.

Referenced by toast_compress_datum().