PostgreSQL Source Code  git master
toast_compression.h File Reference
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros

#define TOAST_PGLZ_COMPRESSION   'p'
 
#define TOAST_LZ4_COMPRESSION   'l'
 
#define InvalidCompressionMethod   '\0'
 
#define CompressionMethodIsValid(cm)   ((cm) != InvalidCompressionMethod)
 
#define IsStorageCompressible(storage)
 

Typedefs

typedef enum ToastCompressionId ToastCompressionId
 

Enumerations

enum  ToastCompressionId { TOAST_PGLZ_COMPRESSION_ID = 0, TOAST_LZ4_COMPRESSION_ID = 1, TOAST_INVALID_COMPRESSION_ID = 2 }
 

Functions

static char GetDefaultToastCompression (void)
 
struct varlenapglz_compress_datum (const struct varlena *value)
 
struct varlenapglz_decompress_datum (const struct varlena *value)
 
struct varlenapglz_decompress_datum_slice (const struct varlena *value, int32 slicelength)
 
struct varlenalz4_compress_datum (const struct varlena *value)
 
struct varlenalz4_decompress_datum (const struct varlena *value)
 
struct varlenalz4_decompress_datum_slice (const struct varlena *value, int32 slicelength)
 
ToastCompressionId toast_get_compression_id (struct varlena *attr)
 
char CompressionNameToMethod (const char *compression)
 
const char * GetCompressionMethodName (char method)
 

Variables

int default_toast_compression
 

Macro Definition Documentation

◆ CompressionMethodIsValid

◆ InvalidCompressionMethod

◆ IsStorageCompressible

#define IsStorageCompressible (   storage)
Value:
((storage) != TYPSTORAGE_PLAIN && \
(storage) != TYPSTORAGE_EXTERNAL)

Definition at line 54 of file toast_compression.h.

Referenced by ATExecAlterColumnType(), ATExecSetCompression(), DefineAttr(), GetAttributeCompression(), and TupleDescInitEntry().

◆ TOAST_LZ4_COMPRESSION

#define TOAST_LZ4_COMPRESSION   'l'

◆ TOAST_PGLZ_COMPRESSION

#define TOAST_PGLZ_COMPRESSION   'p'

Typedef Documentation

◆ ToastCompressionId

Enumeration Type Documentation

◆ ToastCompressionId

Enumerator
TOAST_PGLZ_COMPRESSION_ID 
TOAST_LZ4_COMPRESSION_ID 
TOAST_INVALID_COMPRESSION_ID 

Definition at line 37 of file toast_compression.h.

Function Documentation

◆ CompressionNameToMethod()

char CompressionNameToMethod ( const char *  compression)

Definition at line 287 of file toast_compression.c.

References InvalidCompressionMethod, NO_LZ4_SUPPORT, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by GetAttributeCompression(), and GetDefaultToastCompression().

288 {
289  if (strcmp(compression, "pglz") == 0)
290  return TOAST_PGLZ_COMPRESSION;
291  else if (strcmp(compression, "lz4") == 0)
292  {
293 #ifndef USE_LZ4
294  NO_LZ4_SUPPORT();
295 #endif
296  return TOAST_LZ4_COMPRESSION;
297  }
298 
300 }
#define NO_LZ4_SUPPORT()
#define TOAST_LZ4_COMPRESSION
#define TOAST_PGLZ_COMPRESSION
#define InvalidCompressionMethod

◆ GetCompressionMethodName()

const char* GetCompressionMethodName ( char  method)

Definition at line 306 of file toast_compression.c.

References elog, ERROR, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by GetDefaultToastCompression(), MergeAttributes(), and transformTableLikeClause().

307 {
308  switch (method)
309  {
311  return "pglz";
313  return "lz4";
314  default:
315  elog(ERROR, "invalid compression method %c", method);
316  return NULL; /* keep compiler quiet */
317  }
318 }
#define ERROR
Definition: elog.h:46
#define TOAST_LZ4_COMPRESSION
#define TOAST_PGLZ_COMPRESSION
#define elog(elevel,...)
Definition: elog.h:232

◆ GetDefaultToastCompression()

◆ lz4_compress_datum()

struct varlena* lz4_compress_datum ( const struct varlena value)

Definition at line 141 of file toast_compression.c.

References elog, ERROR, NO_LZ4_SUPPORT, palloc(), pfree(), SET_VARSIZE_COMPRESSED, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by GetDefaultToastCompression(), and toast_compress_datum().

142 {
143 #ifndef USE_LZ4
144  NO_LZ4_SUPPORT();
145  return NULL; /* keep compiler quiet */
146 #else
147  int32 valsize;
148  int32 len;
149  int32 max_size;
150  struct varlena *tmp = NULL;
151 
152  valsize = VARSIZE_ANY_EXHDR(value);
153 
154  /*
155  * Figure out the maximum possible size of the LZ4 output, add the bytes
156  * that will be needed for varlena overhead, and allocate that amount.
157  */
158  max_size = LZ4_compressBound(valsize);
159  tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
160 
161  len = LZ4_compress_default(VARDATA_ANY(value),
162  (char *) tmp + VARHDRSZ_COMPRESSED,
163  valsize, max_size);
164  if (len <= 0)
165  elog(ERROR, "lz4 compression failed");
166 
167  /* data is incompressible so just free the memory and return NULL */
168  if (len > valsize)
169  {
170  pfree(tmp);
171  return NULL;
172  }
173 
175 
176  return tmp;
177 #endif
178 }
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
#define NO_LZ4_SUPPORT()
signed int int32
Definition: c.h:429
void pfree(void *pointer)
Definition: mcxt.c:1169
#define ERROR
Definition: elog.h:46
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
void * palloc(Size size)
Definition: mcxt.c:1062
#define elog(elevel,...)
Definition: elog.h:232
#define SET_VARSIZE_COMPRESSED(PTR, len)
Definition: postgres.h:344
Definition: c.h:621
#define VARHDRSZ_COMPRESSED
Definition: postgres.h:291

◆ lz4_decompress_datum()

struct varlena* lz4_decompress_datum ( const struct varlena value)

Definition at line 184 of file toast_compression.c.

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by GetDefaultToastCompression(), lz4_decompress_datum_slice(), and toast_decompress_datum().

185 {
186 #ifndef USE_LZ4
187  NO_LZ4_SUPPORT();
188  return NULL; /* keep compiler quiet */
189 #else
190  int32 rawsize;
191  struct varlena *result;
192 
193  /* allocate memory for the uncompressed data */
194  result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
195 
196  /* decompress the data */
197  rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESSED,
198  VARDATA(result),
199  VARSIZE(value) - VARHDRSZ_COMPRESSED,
201  if (rawsize < 0)
202  ereport(ERROR,
204  errmsg_internal("compressed lz4 data is corrupt")));
205 
206 
207  SET_VARSIZE(result, rawsize + VARHDRSZ);
208 
209  return result;
210 #endif
211 }
#define VARDATA(PTR)
Definition: postgres.h:315
#define VARSIZE(PTR)
Definition: postgres.h:316
#define VARHDRSZ
Definition: c.h:627
#define NO_LZ4_SUPPORT()
int errcode(int sqlerrcode)
Definition: elog.c:698
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR)
Definition: postgres.h:365
signed int int32
Definition: c.h:429
#define ERROR
Definition: elog.h:46
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
#define ereport(elevel,...)
Definition: elog.h:157
int errmsg_internal(const char *fmt,...)
Definition: elog.c:996
void * palloc(Size size)
Definition: mcxt.c:1062
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
#define VARHDRSZ_COMPRESSED
Definition: postgres.h:291

◆ lz4_decompress_datum_slice()

struct varlena* lz4_decompress_datum_slice ( const struct varlena value,
int32  slicelength 
)

Definition at line 217 of file toast_compression.c.

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, lz4_decompress_datum(), NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by GetDefaultToastCompression(), and toast_decompress_datum_slice().

218 {
219 #ifndef USE_LZ4
220  NO_LZ4_SUPPORT();
221  return NULL; /* keep compiler quiet */
222 #else
223  int32 rawsize;
224  struct varlena *result;
225 
226  /* slice decompression not supported prior to 1.8.3 */
227  if (LZ4_versionNumber() < 10803)
228  return lz4_decompress_datum(value);
229 
230  /* allocate memory for the uncompressed data */
231  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
232 
233  /* decompress the data */
234  rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESSED,
235  VARDATA(result),
236  VARSIZE(value) - VARHDRSZ_COMPRESSED,
237  slicelength,
238  slicelength);
239  if (rawsize < 0)
240  ereport(ERROR,
242  errmsg_internal("compressed lz4 data is corrupt")));
243 
244  SET_VARSIZE(result, rawsize + VARHDRSZ);
245 
246  return result;
247 #endif
248 }
#define VARDATA(PTR)
Definition: postgres.h:315
#define VARSIZE(PTR)
Definition: postgres.h:316
#define VARHDRSZ
Definition: c.h:627
#define NO_LZ4_SUPPORT()
int errcode(int sqlerrcode)
Definition: elog.c:698
signed int int32
Definition: c.h:429
#define ERROR
Definition: elog.h:46
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
#define ereport(elevel,...)
Definition: elog.h:157
int errmsg_internal(const char *fmt,...)
Definition: elog.c:996
void * palloc(Size size)
Definition: mcxt.c:1062
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
struct varlena * lz4_decompress_datum(const struct varlena *value)
#define VARHDRSZ_COMPRESSED
Definition: postgres.h:291

◆ pglz_compress_datum()

struct varlena* pglz_compress_datum ( const struct varlena value)

Definition at line 42 of file toast_compression.c.

References DatumGetPointer, PGLZ_Strategy::max_input_size, palloc(), pfree(), pglz_compress(), PGLZ_MAX_OUTPUT, PGLZ_strategy_default, SET_VARSIZE_COMPRESSED, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by GetDefaultToastCompression(), and toast_compress_datum().

43 {
44  int32 valsize,
45  len;
46  struct varlena *tmp = NULL;
47 
48  valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
49 
50  /*
51  * No point in wasting a palloc cycle if value size is outside the allowed
52  * range for compression.
53  */
54  if (valsize < PGLZ_strategy_default->min_input_size ||
56  return NULL;
57 
58  /*
59  * Figure out the maximum possible size of the pglz output, add the bytes
60  * that will be needed for varlena overhead, and allocate that amount.
61  */
62  tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
64 
65  len = pglz_compress(VARDATA_ANY(value),
66  valsize,
67  (char *) tmp + VARHDRSZ_COMPRESSED,
68  NULL);
69  if (len < 0)
70  {
71  pfree(tmp);
72  return NULL;
73  }
74 
76 
77  return tmp;
78 }
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
signed int int32
Definition: c.h:429
void pfree(void *pointer)
Definition: mcxt.c:1169
int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy)
#define PGLZ_MAX_OUTPUT(_dlen)
Definition: pg_lzcompress.h:21
int32 max_input_size
Definition: pg_lzcompress.h:60
#define DatumGetPointer(X)
Definition: postgres.h:593
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
void * palloc(Size size)
Definition: mcxt.c:1062
#define SET_VARSIZE_COMPRESSED(PTR, len)
Definition: postgres.h:344
Definition: c.h:621
const PGLZ_Strategy *const PGLZ_strategy_default
#define VARHDRSZ_COMPRESSED
Definition: postgres.h:291

◆ pglz_decompress_datum()

struct varlena* pglz_decompress_datum ( const struct varlena value)

Definition at line 84 of file toast_compression.c.

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by GetDefaultToastCompression(), and toast_decompress_datum().

85 {
86  struct varlena *result;
87  int32 rawsize;
88 
89  /* allocate memory for the uncompressed data */
90  result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
91 
92  /* decompress the data */
93  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
95  VARDATA(result),
96  VARDATA_COMPRESSED_GET_EXTSIZE(value), true);
97  if (rawsize < 0)
98  ereport(ERROR,
100  errmsg_internal("compressed pglz data is corrupt")));
101 
102  SET_VARSIZE(result, rawsize + VARHDRSZ);
103 
104  return result;
105 }
#define VARDATA(PTR)
Definition: postgres.h:315
#define VARSIZE(PTR)
Definition: postgres.h:316
#define VARHDRSZ
Definition: c.h:627
int errcode(int sqlerrcode)
Definition: elog.c:698
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR)
Definition: postgres.h:365
int32 pglz_decompress(const char *source, int32 slen, char *dest, int32 rawsize, bool check_complete)
signed int int32
Definition: c.h:429
#define ERROR
Definition: elog.h:46
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
#define ereport(elevel,...)
Definition: elog.h:157
int errmsg_internal(const char *fmt,...)
Definition: elog.c:996
void * palloc(Size size)
Definition: mcxt.c:1062
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
#define VARHDRSZ_COMPRESSED
Definition: postgres.h:291

◆ pglz_decompress_datum_slice()

struct varlena* pglz_decompress_datum_slice ( const struct varlena value,
int32  slicelength 
)

Definition at line 111 of file toast_compression.c.

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by GetDefaultToastCompression(), and toast_decompress_datum_slice().

113 {
114  struct varlena *result;
115  int32 rawsize;
116 
117  /* allocate memory for the uncompressed data */
118  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
119 
120  /* decompress the data */
121  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
122  VARSIZE(value) - VARHDRSZ_COMPRESSED,
123  VARDATA(result),
124  slicelength, false);
125  if (rawsize < 0)
126  ereport(ERROR,
128  errmsg_internal("compressed pglz data is corrupt")));
129 
130  SET_VARSIZE(result, rawsize + VARHDRSZ);
131 
132  return result;
133 }
#define VARDATA(PTR)
Definition: postgres.h:315
#define VARSIZE(PTR)
Definition: postgres.h:316
#define VARHDRSZ
Definition: c.h:627
int errcode(int sqlerrcode)
Definition: elog.c:698
int32 pglz_decompress(const char *source, int32 slen, char *dest, int32 rawsize, bool check_complete)
signed int int32
Definition: c.h:429
#define ERROR
Definition: elog.h:46
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
#define ereport(elevel,...)
Definition: elog.h:157
int errmsg_internal(const char *fmt,...)
Definition: elog.c:996
void * palloc(Size size)
Definition: mcxt.c:1062
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
#define VARHDRSZ_COMPRESSED
Definition: postgres.h:291

◆ toast_get_compression_id()

ToastCompressionId toast_get_compression_id ( struct varlena attr)

Definition at line 256 of file toast_compression.c.

References TOAST_INVALID_COMPRESSION_ID, VARATT_EXTERNAL_GET_COMPRESS_METHOD, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL_ONDISK, and VARDATA_COMPRESSED_GET_COMPRESS_METHOD.

Referenced by GetDefaultToastCompression(), pg_column_compression(), and reform_and_rewrite_tuple().

257 {
259 
260  /*
261  * If it is stored externally then fetch the compression method id from
262  * the external toast pointer. If compressed inline, fetch it from the
263  * toast compression header.
264  */
265  if (VARATT_IS_EXTERNAL_ONDISK(attr))
266  {
267  struct varatt_external toast_pointer;
268 
269  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
270 
271  if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
272  cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
273  }
274  else if (VARATT_IS_COMPRESSED(attr))
276 
277  return cmid;
278 }
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: postgres.h:327
#define VARATT_IS_COMPRESSED(PTR)
Definition: postgres.h:325
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)
Definition: postgres.h:391
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
ToastCompressionId
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer)
Definition: postgres.h:373
#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR)
Definition: postgres.h:367

Variable Documentation

◆ default_toast_compression

int default_toast_compression

Definition at line 27 of file toast_compression.c.

Referenced by GetDefaultToastCompression().