PostgreSQL Source Code  git master
toast_compression.h File Reference
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros

#define TOAST_PGLZ_COMPRESSION   'p'
 
#define TOAST_LZ4_COMPRESSION   'l'
 
#define InvalidCompressionMethod   '\0'
 
#define CompressionMethodIsValid(cm)   ((cm) != InvalidCompressionMethod)
 

Typedefs

typedef enum ToastCompressionId ToastCompressionId
 

Enumerations

enum  ToastCompressionId { TOAST_PGLZ_COMPRESSION_ID = 0 , TOAST_LZ4_COMPRESSION_ID = 1 , TOAST_INVALID_COMPRESSION_ID = 2 }
 

Functions

struct varlenapglz_compress_datum (const struct varlena *value)
 
struct varlenapglz_decompress_datum (const struct varlena *value)
 
struct varlenapglz_decompress_datum_slice (const struct varlena *value, int32 slicelength)
 
struct varlenalz4_compress_datum (const struct varlena *value)
 
struct varlenalz4_decompress_datum (const struct varlena *value)
 
struct varlenalz4_decompress_datum_slice (const struct varlena *value, int32 slicelength)
 
ToastCompressionId toast_get_compression_id (struct varlena *attr)
 
char CompressionNameToMethod (const char *compression)
 
const char * GetCompressionMethodName (char method)
 

Variables

PGDLLIMPORT int default_toast_compression
 

Macro Definition Documentation

◆ CompressionMethodIsValid

#define CompressionMethodIsValid (   cm)    ((cm) != InvalidCompressionMethod)

Definition at line 53 of file toast_compression.h.

◆ InvalidCompressionMethod

#define InvalidCompressionMethod   '\0'

Definition at line 51 of file toast_compression.h.

◆ TOAST_LZ4_COMPRESSION

#define TOAST_LZ4_COMPRESSION   'l'

Definition at line 50 of file toast_compression.h.

◆ TOAST_PGLZ_COMPRESSION

#define TOAST_PGLZ_COMPRESSION   'p'

Definition at line 49 of file toast_compression.h.

Typedef Documentation

◆ ToastCompressionId

Enumeration Type Documentation

◆ ToastCompressionId

Enumerator
TOAST_PGLZ_COMPRESSION_ID 
TOAST_LZ4_COMPRESSION_ID 
TOAST_INVALID_COMPRESSION_ID 

Definition at line 37 of file toast_compression.h.

38 {
ToastCompressionId
@ TOAST_INVALID_COMPRESSION_ID
@ TOAST_LZ4_COMPRESSION_ID
@ TOAST_PGLZ_COMPRESSION_ID

Function Documentation

◆ CompressionNameToMethod()

char CompressionNameToMethod ( const char *  compression)

Definition at line 286 of file toast_compression.c.

287 {
288  if (strcmp(compression, "pglz") == 0)
289  return TOAST_PGLZ_COMPRESSION;
290  else if (strcmp(compression, "lz4") == 0)
291  {
292 #ifndef USE_LZ4
293  NO_LZ4_SUPPORT();
294 #endif
295  return TOAST_LZ4_COMPRESSION;
296  }
297 
299 }
#define NO_LZ4_SUPPORT()
#define TOAST_PGLZ_COMPRESSION
#define InvalidCompressionMethod
#define TOAST_LZ4_COMPRESSION

References InvalidCompressionMethod, NO_LZ4_SUPPORT, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by GetAttributeCompression().

◆ GetCompressionMethodName()

const char* GetCompressionMethodName ( char  method)

Definition at line 305 of file toast_compression.c.

306 {
307  switch (method)
308  {
310  return "pglz";
312  return "lz4";
313  default:
314  elog(ERROR, "invalid compression method %c", method);
315  return NULL; /* keep compiler quiet */
316  }
317 }
#define ERROR
Definition: elog.h:33

References elog(), ERROR, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by MergeAttributes(), and transformTableLikeClause().

◆ lz4_compress_datum()

struct varlena* lz4_compress_datum ( const struct varlena value)

Definition at line 140 of file toast_compression.c.

141 {
142 #ifndef USE_LZ4
143  NO_LZ4_SUPPORT();
144  return NULL; /* keep compiler quiet */
145 #else
146  int32 valsize;
147  int32 len;
148  int32 max_size;
149  struct varlena *tmp = NULL;
150 
151  valsize = VARSIZE_ANY_EXHDR(value);
152 
153  /*
154  * Figure out the maximum possible size of the LZ4 output, add the bytes
155  * that will be needed for varlena overhead, and allocate that amount.
156  */
157  max_size = LZ4_compressBound(valsize);
158  tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
159 
160  len = LZ4_compress_default(VARDATA_ANY(value),
161  (char *) tmp + VARHDRSZ_COMPRESSED,
162  valsize, max_size);
163  if (len <= 0)
164  elog(ERROR, "lz4 compression failed");
165 
166  /* data is incompressible so just free the memory and return NULL */
167  if (len > valsize)
168  {
169  pfree(tmp);
170  return NULL;
171  }
172 
174 
175  return tmp;
176 #endif
177 }
signed int int32
Definition: c.h:440
static struct @151 value
void pfree(void *pointer)
Definition: mcxt.c:1175
void * palloc(Size size)
Definition: mcxt.c:1068
const void size_t len
#define SET_VARSIZE_COMPRESSED(PTR, len)
Definition: postgres.h:344
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
#define VARHDRSZ_COMPRESSED
Definition: postgres.h:291
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
Definition: c.h:633

References elog(), ERROR, len, NO_LZ4_SUPPORT, palloc(), pfree(), SET_VARSIZE_COMPRESSED, value, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by toast_compress_datum().

◆ lz4_decompress_datum()

struct varlena* lz4_decompress_datum ( const struct varlena value)

Definition at line 183 of file toast_compression.c.

184 {
185 #ifndef USE_LZ4
186  NO_LZ4_SUPPORT();
187  return NULL; /* keep compiler quiet */
188 #else
189  int32 rawsize;
190  struct varlena *result;
191 
192  /* allocate memory for the uncompressed data */
194 
195  /* decompress the data */
196  rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESSED,
197  VARDATA(result),
200  if (rawsize < 0)
201  ereport(ERROR,
203  errmsg_internal("compressed lz4 data is corrupt")));
204 
205 
206  SET_VARSIZE(result, rawsize + VARHDRSZ);
207 
208  return result;
209 #endif
210 }
#define VARHDRSZ
Definition: c.h:638
int errmsg_internal(const char *fmt,...)
Definition: elog.c:991
int errcode(int sqlerrcode)
Definition: elog.c:693
#define ereport(elevel,...)
Definition: elog.h:143
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:43
#define VARDATA(PTR)
Definition: postgres.h:315
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR)
Definition: postgres.h:365
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
#define VARSIZE(PTR)
Definition: postgres.h:316

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, value, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by lz4_decompress_datum_slice(), and toast_decompress_datum().

◆ lz4_decompress_datum_slice()

struct varlena* lz4_decompress_datum_slice ( const struct varlena value,
int32  slicelength 
)

Definition at line 216 of file toast_compression.c.

217 {
218 #ifndef USE_LZ4
219  NO_LZ4_SUPPORT();
220  return NULL; /* keep compiler quiet */
221 #else
222  int32 rawsize;
223  struct varlena *result;
224 
225  /* slice decompression not supported prior to 1.8.3 */
226  if (LZ4_versionNumber() < 10803)
227  return lz4_decompress_datum(value);
228 
229  /* allocate memory for the uncompressed data */
230  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
231 
232  /* decompress the data */
233  rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESSED,
234  VARDATA(result),
236  slicelength,
237  slicelength);
238  if (rawsize < 0)
239  ereport(ERROR,
241  errmsg_internal("compressed lz4 data is corrupt")));
242 
243  SET_VARSIZE(result, rawsize + VARHDRSZ);
244 
245  return result;
246 #endif
247 }
struct varlena * lz4_decompress_datum(const struct varlena *value)

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, lz4_decompress_datum(), NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, value, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum_slice().

◆ pglz_compress_datum()

struct varlena* pglz_compress_datum ( const struct varlena value)

Definition at line 41 of file toast_compression.c.

42 {
43  int32 valsize,
44  len;
45  struct varlena *tmp = NULL;
46 
48 
49  /*
50  * No point in wasting a palloc cycle if value size is outside the allowed
51  * range for compression.
52  */
53  if (valsize < PGLZ_strategy_default->min_input_size ||
55  return NULL;
56 
57  /*
58  * Figure out the maximum possible size of the pglz output, add the bytes
59  * that will be needed for varlena overhead, and allocate that amount.
60  */
61  tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
63 
65  valsize,
66  (char *) tmp + VARHDRSZ_COMPRESSED,
67  NULL);
68  if (len < 0)
69  {
70  pfree(tmp);
71  return NULL;
72  }
73 
75 
76  return tmp;
77 }
const PGLZ_Strategy *const PGLZ_strategy_default
int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy)
#define PGLZ_MAX_OUTPUT(_dlen)
Definition: pg_lzcompress.h:21
#define DatumGetPointer(X)
Definition: postgres.h:593
int32 max_input_size
Definition: pg_lzcompress.h:60

References DatumGetPointer, len, PGLZ_Strategy::max_input_size, palloc(), pfree(), pglz_compress(), PGLZ_MAX_OUTPUT, PGLZ_strategy_default, SET_VARSIZE_COMPRESSED, value, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by toast_compress_datum().

◆ pglz_decompress_datum()

struct varlena* pglz_decompress_datum ( const struct varlena value)

Definition at line 83 of file toast_compression.c.

84 {
85  struct varlena *result;
86  int32 rawsize;
87 
88  /* allocate memory for the uncompressed data */
90 
91  /* decompress the data */
92  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
94  VARDATA(result),
96  if (rawsize < 0)
97  ereport(ERROR,
99  errmsg_internal("compressed pglz data is corrupt")));
100 
101  SET_VARSIZE(result, rawsize + VARHDRSZ);
102 
103  return result;
104 }
int32 pglz_decompress(const char *source, int32 slen, char *dest, int32 rawsize, bool check_complete)

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, value, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum().

◆ pglz_decompress_datum_slice()

struct varlena* pglz_decompress_datum_slice ( const struct varlena value,
int32  slicelength 
)

Definition at line 110 of file toast_compression.c.

112 {
113  struct varlena *result;
114  int32 rawsize;
115 
116  /* allocate memory for the uncompressed data */
117  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
118 
119  /* decompress the data */
120  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
122  VARDATA(result),
123  slicelength, false);
124  if (rawsize < 0)
125  ereport(ERROR,
127  errmsg_internal("compressed pglz data is corrupt")));
128 
129  SET_VARSIZE(result, rawsize + VARHDRSZ);
130 
131  return result;
132 }

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, value, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum_slice().

◆ toast_get_compression_id()

ToastCompressionId toast_get_compression_id ( struct varlena attr)

Definition at line 255 of file toast_compression.c.

256 {
258 
259  /*
260  * If it is stored externally then fetch the compression method id from
261  * the external toast pointer. If compressed inline, fetch it from the
262  * toast compression header.
263  */
264  if (VARATT_IS_EXTERNAL_ONDISK(attr))
265  {
266  struct varatt_external toast_pointer;
267 
268  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
269 
270  if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
271  cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
272  }
273  else if (VARATT_IS_COMPRESSED(attr))
275 
276  return cmid;
277 }
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: postgres.h:327
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer)
Definition: postgres.h:373
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)
Definition: postgres.h:391
#define VARATT_IS_COMPRESSED(PTR)
Definition: postgres.h:325
#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR)
Definition: postgres.h:367

References TOAST_INVALID_COMPRESSION_ID, VARATT_EXTERNAL_GET_COMPRESS_METHOD, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL_ONDISK, and VARDATA_COMPRESSED_GET_COMPRESS_METHOD.

Referenced by pg_column_compression().

Variable Documentation

◆ default_toast_compression

PGDLLIMPORT int default_toast_compression
extern

Definition at line 27 of file toast_compression.c.

Referenced by toast_compress_datum().