PostgreSQL Source Code  git master
toast_compression.h File Reference
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros

#define TOAST_PGLZ_COMPRESSION   'p'
 
#define TOAST_LZ4_COMPRESSION   'l'
 
#define InvalidCompressionMethod   '\0'
 
#define CompressionMethodIsValid(cm)   ((cm) != InvalidCompressionMethod)
 

Typedefs

typedef enum ToastCompressionId ToastCompressionId
 

Enumerations

enum  ToastCompressionId { TOAST_PGLZ_COMPRESSION_ID = 0 , TOAST_LZ4_COMPRESSION_ID = 1 , TOAST_INVALID_COMPRESSION_ID = 2 }
 

Functions

struct varlenapglz_compress_datum (const struct varlena *value)
 
struct varlenapglz_decompress_datum (const struct varlena *value)
 
struct varlenapglz_decompress_datum_slice (const struct varlena *value, int32 slicelength)
 
struct varlenalz4_compress_datum (const struct varlena *value)
 
struct varlenalz4_decompress_datum (const struct varlena *value)
 
struct varlenalz4_decompress_datum_slice (const struct varlena *value, int32 slicelength)
 
ToastCompressionId toast_get_compression_id (struct varlena *attr)
 
char CompressionNameToMethod (const char *compression)
 
const char * GetCompressionMethodName (char method)
 

Variables

PGDLLIMPORT int default_toast_compression
 

Macro Definition Documentation

◆ CompressionMethodIsValid

#define CompressionMethodIsValid (   cm)    ((cm) != InvalidCompressionMethod)

Definition at line 53 of file toast_compression.h.

◆ InvalidCompressionMethod

#define InvalidCompressionMethod   '\0'

Definition at line 51 of file toast_compression.h.

◆ TOAST_LZ4_COMPRESSION

#define TOAST_LZ4_COMPRESSION   'l'

Definition at line 50 of file toast_compression.h.

◆ TOAST_PGLZ_COMPRESSION

#define TOAST_PGLZ_COMPRESSION   'p'

Definition at line 49 of file toast_compression.h.

Typedef Documentation

◆ ToastCompressionId

Enumeration Type Documentation

◆ ToastCompressionId

Enumerator
TOAST_PGLZ_COMPRESSION_ID 
TOAST_LZ4_COMPRESSION_ID 
TOAST_INVALID_COMPRESSION_ID 

Definition at line 37 of file toast_compression.h.

38 {
ToastCompressionId
@ TOAST_INVALID_COMPRESSION_ID
@ TOAST_LZ4_COMPRESSION_ID
@ TOAST_PGLZ_COMPRESSION_ID

Function Documentation

◆ CompressionNameToMethod()

char CompressionNameToMethod ( const char *  compression)

Definition at line 285 of file toast_compression.c.

286 {
287  if (strcmp(compression, "pglz") == 0)
288  return TOAST_PGLZ_COMPRESSION;
289  else if (strcmp(compression, "lz4") == 0)
290  {
291 #ifndef USE_LZ4
292  NO_LZ4_SUPPORT();
293 #endif
294  return TOAST_LZ4_COMPRESSION;
295  }
296 
298 }
#define NO_LZ4_SUPPORT()
#define TOAST_PGLZ_COMPRESSION
#define InvalidCompressionMethod
#define TOAST_LZ4_COMPRESSION

References InvalidCompressionMethod, NO_LZ4_SUPPORT, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by GetAttributeCompression().

◆ GetCompressionMethodName()

const char* GetCompressionMethodName ( char  method)

Definition at line 304 of file toast_compression.c.

305 {
306  switch (method)
307  {
309  return "pglz";
311  return "lz4";
312  default:
313  elog(ERROR, "invalid compression method %c", method);
314  return NULL; /* keep compiler quiet */
315  }
316 }
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:224

References elog, ERROR, TOAST_LZ4_COMPRESSION, and TOAST_PGLZ_COMPRESSION.

Referenced by MergeAttributes(), and transformTableLikeClause().

◆ lz4_compress_datum()

struct varlena* lz4_compress_datum ( const struct varlena value)

Definition at line 139 of file toast_compression.c.

140 {
141 #ifndef USE_LZ4
142  NO_LZ4_SUPPORT();
143  return NULL; /* keep compiler quiet */
144 #else
145  int32 valsize;
146  int32 len;
147  int32 max_size;
148  struct varlena *tmp = NULL;
149 
150  valsize = VARSIZE_ANY_EXHDR(value);
151 
152  /*
153  * Figure out the maximum possible size of the LZ4 output, add the bytes
154  * that will be needed for varlena overhead, and allocate that amount.
155  */
156  max_size = LZ4_compressBound(valsize);
157  tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
158 
159  len = LZ4_compress_default(VARDATA_ANY(value),
160  (char *) tmp + VARHDRSZ_COMPRESSED,
161  valsize, max_size);
162  if (len <= 0)
163  elog(ERROR, "lz4 compression failed");
164 
165  /* data is incompressible so just free the memory and return NULL */
166  if (len > valsize)
167  {
168  pfree(tmp);
169  return NULL;
170  }
171 
173 
174  return tmp;
175 #endif
176 }
signed int int32
Definition: c.h:481
static struct @149 value
void pfree(void *pointer)
Definition: mcxt.c:1401
void * palloc(Size size)
Definition: mcxt.c:1197
const void size_t len
Definition: c.h:674
#define SET_VARSIZE_COMPRESSED(PTR, len)
Definition: varatt.h:307
#define VARDATA_ANY(PTR)
Definition: varatt.h:324
#define VARHDRSZ_COMPRESSED
Definition: varatt.h:254
#define VARSIZE_ANY_EXHDR(PTR)
Definition: varatt.h:317

References elog, ERROR, len, NO_LZ4_SUPPORT, palloc(), pfree(), SET_VARSIZE_COMPRESSED, value, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by toast_compress_datum().

◆ lz4_decompress_datum()

struct varlena* lz4_decompress_datum ( const struct varlena value)

Definition at line 182 of file toast_compression.c.

183 {
184 #ifndef USE_LZ4
185  NO_LZ4_SUPPORT();
186  return NULL; /* keep compiler quiet */
187 #else
188  int32 rawsize;
189  struct varlena *result;
190 
191  /* allocate memory for the uncompressed data */
193 
194  /* decompress the data */
195  rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESSED,
196  VARDATA(result),
199  if (rawsize < 0)
200  ereport(ERROR,
202  errmsg_internal("compressed lz4 data is corrupt")));
203 
204 
205  SET_VARSIZE(result, rawsize + VARHDRSZ);
206 
207  return result;
208 #endif
209 }
#define VARHDRSZ
Definition: c.h:679
int errmsg_internal(const char *fmt,...)
Definition: elog.c:1159
int errcode(int sqlerrcode)
Definition: elog.c:859
#define ereport(elevel,...)
Definition: elog.h:149
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:41
#define VARDATA(PTR)
Definition: varatt.h:278
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR)
Definition: varatt.h:328
#define SET_VARSIZE(PTR, len)
Definition: varatt.h:305
#define VARSIZE(PTR)
Definition: varatt.h:279

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, value, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by lz4_decompress_datum_slice(), and toast_decompress_datum().

◆ lz4_decompress_datum_slice()

struct varlena* lz4_decompress_datum_slice ( const struct varlena value,
int32  slicelength 
)

Definition at line 215 of file toast_compression.c.

216 {
217 #ifndef USE_LZ4
218  NO_LZ4_SUPPORT();
219  return NULL; /* keep compiler quiet */
220 #else
221  int32 rawsize;
222  struct varlena *result;
223 
224  /* slice decompression not supported prior to 1.8.3 */
225  if (LZ4_versionNumber() < 10803)
226  return lz4_decompress_datum(value);
227 
228  /* allocate memory for the uncompressed data */
229  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
230 
231  /* decompress the data */
232  rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESSED,
233  VARDATA(result),
235  slicelength,
236  slicelength);
237  if (rawsize < 0)
238  ereport(ERROR,
240  errmsg_internal("compressed lz4 data is corrupt")));
241 
242  SET_VARSIZE(result, rawsize + VARHDRSZ);
243 
244  return result;
245 #endif
246 }
struct varlena * lz4_decompress_datum(const struct varlena *value)

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, lz4_decompress_datum(), NO_LZ4_SUPPORT, palloc(), SET_VARSIZE, value, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum_slice().

◆ pglz_compress_datum()

struct varlena* pglz_compress_datum ( const struct varlena value)

Definition at line 40 of file toast_compression.c.

41 {
42  int32 valsize,
43  len;
44  struct varlena *tmp = NULL;
45 
46  valsize = VARSIZE_ANY_EXHDR(value);
47 
48  /*
49  * No point in wasting a palloc cycle if value size is outside the allowed
50  * range for compression.
51  */
52  if (valsize < PGLZ_strategy_default->min_input_size ||
54  return NULL;
55 
56  /*
57  * Figure out the maximum possible size of the pglz output, add the bytes
58  * that will be needed for varlena overhead, and allocate that amount.
59  */
60  tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
62 
64  valsize,
65  (char *) tmp + VARHDRSZ_COMPRESSED,
66  NULL);
67  if (len < 0)
68  {
69  pfree(tmp);
70  return NULL;
71  }
72 
74 
75  return tmp;
76 }
const PGLZ_Strategy *const PGLZ_strategy_default
int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy)
#define PGLZ_MAX_OUTPUT(_dlen)
Definition: pg_lzcompress.h:21
int32 max_input_size
Definition: pg_lzcompress.h:60

References len, PGLZ_Strategy::max_input_size, palloc(), pfree(), pglz_compress(), PGLZ_MAX_OUTPUT, PGLZ_strategy_default, SET_VARSIZE_COMPRESSED, value, VARDATA_ANY, VARHDRSZ_COMPRESSED, and VARSIZE_ANY_EXHDR.

Referenced by toast_compress_datum().

◆ pglz_decompress_datum()

struct varlena* pglz_decompress_datum ( const struct varlena value)

Definition at line 82 of file toast_compression.c.

83 {
84  struct varlena *result;
85  int32 rawsize;
86 
87  /* allocate memory for the uncompressed data */
89 
90  /* decompress the data */
91  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
93  VARDATA(result),
95  if (rawsize < 0)
96  ereport(ERROR,
98  errmsg_internal("compressed pglz data is corrupt")));
99 
100  SET_VARSIZE(result, rawsize + VARHDRSZ);
101 
102  return result;
103 }
int32 pglz_decompress(const char *source, int32 slen, char *dest, int32 rawsize, bool check_complete)

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, value, VARDATA, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum().

◆ pglz_decompress_datum_slice()

struct varlena* pglz_decompress_datum_slice ( const struct varlena value,
int32  slicelength 
)

Definition at line 109 of file toast_compression.c.

111 {
112  struct varlena *result;
113  int32 rawsize;
114 
115  /* allocate memory for the uncompressed data */
116  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
117 
118  /* decompress the data */
119  rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
121  VARDATA(result),
122  slicelength, false);
123  if (rawsize < 0)
124  ereport(ERROR,
126  errmsg_internal("compressed pglz data is corrupt")));
127 
128  SET_VARSIZE(result, rawsize + VARHDRSZ);
129 
130  return result;
131 }

References ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg_internal(), ERROR, palloc(), pglz_decompress(), SET_VARSIZE, value, VARDATA, VARHDRSZ, VARHDRSZ_COMPRESSED, and VARSIZE.

Referenced by toast_decompress_datum_slice().

◆ toast_get_compression_id()

ToastCompressionId toast_get_compression_id ( struct varlena attr)

Definition at line 254 of file toast_compression.c.

255 {
257 
258  /*
259  * If it is stored externally then fetch the compression method id from
260  * the external toast pointer. If compressed inline, fetch it from the
261  * toast compression header.
262  */
263  if (VARATT_IS_EXTERNAL_ONDISK(attr))
264  {
265  struct varatt_external toast_pointer;
266 
267  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
268 
269  if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
270  cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
271  }
272  else if (VARATT_IS_COMPRESSED(attr))
274 
275  return cmid;
276 }
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: varatt.h:290
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer)
Definition: varatt.h:336
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)
Definition: varatt.h:354
#define VARATT_IS_COMPRESSED(PTR)
Definition: varatt.h:288
#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR)
Definition: varatt.h:330

References TOAST_INVALID_COMPRESSION_ID, VARATT_EXTERNAL_GET_COMPRESS_METHOD, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL_ONDISK, and VARDATA_COMPRESSED_GET_COMPRESS_METHOD.

Referenced by pg_column_compression().

Variable Documentation

◆ default_toast_compression

PGDLLIMPORT int default_toast_compression
extern

Definition at line 26 of file toast_compression.c.

Referenced by toast_compress_datum().