PostgreSQL Source Code  git master
detoast.c File Reference
#include "postgres.h"
#include "access/detoast.h"
#include "access/table.h"
#include "access/tableam.h"
#include "access/toast_internals.h"
#include "common/int.h"
#include "common/pg_lzcompress.h"
#include "utils/expandeddatum.h"
#include "utils/rel.h"
Include dependency graph for detoast.c:

Go to the source code of this file.

Functions

static struct varlenatoast_fetch_datum (struct varlena *attr)
 
static struct varlenatoast_fetch_datum_slice (struct varlena *attr, int32 sliceoffset, int32 slicelength)
 
static struct varlenatoast_decompress_datum (struct varlena *attr)
 
static struct varlenatoast_decompress_datum_slice (struct varlena *attr, int32 slicelength)
 
struct varlenadetoast_external_attr (struct varlena *attr)
 
struct varlenadetoast_attr (struct varlena *attr)
 
struct varlenadetoast_attr_slice (struct varlena *attr, int32 sliceoffset, int32 slicelength)
 
Size toast_raw_datum_size (Datum value)
 
Size toast_datum_size (Datum value)
 

Function Documentation

◆ detoast_attr()

struct varlena* detoast_attr ( struct varlena attr)

Definition at line 116 of file detoast.c.

References Assert, detoast_attr(), detoast_external_attr(), palloc(), pfree(), varatt_indirect::pointer, SET_VARSIZE, toast_decompress_datum(), toast_fetch_datum(), VARATT_EXTERNAL_GET_POINTER, VARATT_IS_COMPRESSED, VARATT_IS_EXTENDED, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARDATA, VARDATA_SHORT, VARHDRSZ, VARHDRSZ_SHORT, VARSIZE_ANY, and VARSIZE_SHORT.

Referenced by detoast_attr(), getdatafield(), pg_detoast_datum(), pg_detoast_datum_copy(), pg_detoast_datum_packed(), toast_flatten_tuple_to_datum(), and toast_tuple_init().

117 {
118  if (VARATT_IS_EXTERNAL_ONDISK(attr))
119  {
120  /*
121  * This is an externally stored datum --- fetch it back from there
122  */
123  attr = toast_fetch_datum(attr);
124  /* If it's compressed, decompress it */
125  if (VARATT_IS_COMPRESSED(attr))
126  {
127  struct varlena *tmp = attr;
128 
129  attr = toast_decompress_datum(tmp);
130  pfree(tmp);
131  }
132  }
133  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
134  {
135  /*
136  * This is an indirect pointer --- dereference it
137  */
138  struct varatt_indirect redirect;
139 
140  VARATT_EXTERNAL_GET_POINTER(redirect, attr);
141  attr = (struct varlena *) redirect.pointer;
142 
143  /* nested indirect Datums aren't allowed */
145 
146  /* recurse in case value is still extended in some other way */
147  attr = detoast_attr(attr);
148 
149  /* if it isn't, we'd better copy it */
150  if (attr == (struct varlena *) redirect.pointer)
151  {
152  struct varlena *result;
153 
154  result = (struct varlena *) palloc(VARSIZE_ANY(attr));
155  memcpy(result, attr, VARSIZE_ANY(attr));
156  attr = result;
157  }
158  }
159  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
160  {
161  /*
162  * This is an expanded-object pointer --- get flat format
163  */
164  attr = detoast_external_attr(attr);
165  /* flatteners are not allowed to produce compressed/short output */
166  Assert(!VARATT_IS_EXTENDED(attr));
167  }
168  else if (VARATT_IS_COMPRESSED(attr))
169  {
170  /*
171  * This is a compressed value inside of the main tuple
172  */
173  attr = toast_decompress_datum(attr);
174  }
175  else if (VARATT_IS_SHORT(attr))
176  {
177  /*
178  * This is a short-header varlena --- convert to 4-byte header format
179  */
180  Size data_size = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT;
181  Size new_size = data_size + VARHDRSZ;
182  struct varlena *new_attr;
183 
184  new_attr = (struct varlena *) palloc(new_size);
185  SET_VARSIZE(new_attr, new_size);
186  memcpy(VARDATA(new_attr), VARDATA_SHORT(attr), data_size);
187  attr = new_attr;
188  }
189 
190  return attr;
191 }
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: postgres.h:327
#define VARATT_IS_COMPRESSED(PTR)
Definition: postgres.h:325
#define VARDATA(PTR)
Definition: postgres.h:315
#define VARATT_IS_EXTERNAL_EXPANDED(PTR)
Definition: postgres.h:335
#define VARHDRSZ_SHORT
Definition: postgres.h:292
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
#define VARHDRSZ
Definition: c.h:627
struct varlena * detoast_external_attr(struct varlena *attr)
Definition: detoast.c:45
struct varlena * detoast_attr(struct varlena *attr)
Definition: detoast.c:116
void pfree(void *pointer)
Definition: mcxt.c:1169
#define VARATT_IS_EXTERNAL_INDIRECT(PTR)
Definition: postgres.h:329
#define VARATT_IS_SHORT(PTR)
Definition: postgres.h:339
#define VARSIZE_SHORT(PTR)
Definition: postgres.h:318
#define VARSIZE_ANY(PTR)
Definition: postgres.h:348
static struct varlena * toast_fetch_datum(struct varlena *attr)
Definition: detoast.c:343
#define Assert(condition)
Definition: c.h:804
size_t Size
Definition: c.h:540
#define VARATT_IS_EXTENDED(PTR)
Definition: postgres.h:340
#define VARDATA_SHORT(PTR)
Definition: postgres.h:319
void * palloc(Size size)
Definition: mcxt.c:1062
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
static struct varlena * toast_decompress_datum(struct varlena *attr)
Definition: detoast.c:471

◆ detoast_attr_slice()

struct varlena* detoast_attr_slice ( struct varlena attr,
int32  sliceoffset,
int32  slicelength 
)

Definition at line 205 of file detoast.c.

References Assert, detoast_attr_slice(), detoast_external_attr(), elog, ERROR, palloc(), pfree(), pg_add_s32_overflow(), pglz_maximum_compressed_size(), varatt_indirect::pointer, SET_VARSIZE, toast_decompress_datum(), toast_decompress_datum_slice(), toast_fetch_datum(), toast_fetch_datum_slice(), TOAST_PGLZ_COMPRESSION_ID, VARATT_EXTERNAL_GET_COMPRESS_METHOD, VARATT_EXTERNAL_GET_EXTSIZE, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARDATA, VARDATA_SHORT, VARHDRSZ, VARHDRSZ_SHORT, VARSIZE, and VARSIZE_SHORT.

Referenced by detoast_attr_slice(), and pg_detoast_datum_slice().

207 {
208  struct varlena *preslice;
209  struct varlena *result;
210  char *attrdata;
211  int32 slicelimit;
212  int32 attrsize;
213 
214  if (sliceoffset < 0)
215  elog(ERROR, "invalid sliceoffset: %d", sliceoffset);
216 
217  /*
218  * Compute slicelimit = offset + length, or -1 if we must fetch all of the
219  * value. In case of integer overflow, we must fetch all.
220  */
221  if (slicelength < 0)
222  slicelimit = -1;
223  else if (pg_add_s32_overflow(sliceoffset, slicelength, &slicelimit))
224  slicelength = slicelimit = -1;
225 
226  if (VARATT_IS_EXTERNAL_ONDISK(attr))
227  {
228  struct varatt_external toast_pointer;
229 
230  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
231 
232  /* fast path for non-compressed external datums */
233  if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
234  return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
235 
236  /*
237  * For compressed values, we need to fetch enough slices to decompress
238  * at least the requested part (when a prefix is requested).
239  * Otherwise, just fetch all slices.
240  */
241  if (slicelimit >= 0)
242  {
243  int32 max_size = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
244 
245  /*
246  * Determine maximum amount of compressed data needed for a prefix
247  * of a given length (after decompression).
248  *
249  * At least for now, if it's LZ4 data, we'll have to fetch the
250  * whole thing, because there doesn't seem to be an API call to
251  * determine how much compressed data we need to be sure of being
252  * able to decompress the required slice.
253  */
254  if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
256  max_size = pglz_maximum_compressed_size(slicelimit, max_size);
257 
258  /*
259  * Fetch enough compressed slices (compressed marker will get set
260  * automatically).
261  */
262  preslice = toast_fetch_datum_slice(attr, 0, max_size);
263  }
264  else
265  preslice = toast_fetch_datum(attr);
266  }
267  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
268  {
269  struct varatt_indirect redirect;
270 
271  VARATT_EXTERNAL_GET_POINTER(redirect, attr);
272 
273  /* nested indirect Datums aren't allowed */
274  Assert(!VARATT_IS_EXTERNAL_INDIRECT(redirect.pointer));
275 
276  return detoast_attr_slice(redirect.pointer,
277  sliceoffset, slicelength);
278  }
279  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
280  {
281  /* pass it off to detoast_external_attr to flatten */
282  preslice = detoast_external_attr(attr);
283  }
284  else
285  preslice = attr;
286 
287  Assert(!VARATT_IS_EXTERNAL(preslice));
288 
289  if (VARATT_IS_COMPRESSED(preslice))
290  {
291  struct varlena *tmp = preslice;
292 
293  /* Decompress enough to encompass the slice and the offset */
294  if (slicelimit >= 0)
295  preslice = toast_decompress_datum_slice(tmp, slicelimit);
296  else
297  preslice = toast_decompress_datum(tmp);
298 
299  if (tmp != attr)
300  pfree(tmp);
301  }
302 
303  if (VARATT_IS_SHORT(preslice))
304  {
305  attrdata = VARDATA_SHORT(preslice);
306  attrsize = VARSIZE_SHORT(preslice) - VARHDRSZ_SHORT;
307  }
308  else
309  {
310  attrdata = VARDATA(preslice);
311  attrsize = VARSIZE(preslice) - VARHDRSZ;
312  }
313 
314  /* slicing of datum for compressed cases and plain value */
315 
316  if (sliceoffset >= attrsize)
317  {
318  sliceoffset = 0;
319  slicelength = 0;
320  }
321  else if (slicelength < 0 || slicelimit > attrsize)
322  slicelength = attrsize - sliceoffset;
323 
324  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
325  SET_VARSIZE(result, slicelength + VARHDRSZ);
326 
327  memcpy(VARDATA(result), attrdata + sliceoffset, slicelength);
328 
329  if (preslice != attr)
330  pfree(preslice);
331 
332  return result;
333 }
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: postgres.h:327
#define VARATT_IS_COMPRESSED(PTR)
Definition: postgres.h:325
int32 pglz_maximum_compressed_size(int32 rawsize, int32 total_compressed_size)
#define VARDATA(PTR)
Definition: postgres.h:315
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)
Definition: postgres.h:391
#define VARATT_IS_EXTERNAL_EXPANDED(PTR)
Definition: postgres.h:335
#define VARHDRSZ_SHORT
Definition: postgres.h:292
#define VARSIZE(PTR)
Definition: postgres.h:316
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
#define VARHDRSZ
Definition: c.h:627
struct varlena * detoast_external_attr(struct varlena *attr)
Definition: detoast.c:45
struct varlena * detoast_attr_slice(struct varlena *attr, int32 sliceoffset, int32 slicelength)
Definition: detoast.c:205
signed int int32
Definition: c.h:429
#define VARATT_IS_EXTERNAL(PTR)
Definition: postgres.h:326
void pfree(void *pointer)
Definition: mcxt.c:1169
#define VARATT_IS_EXTERNAL_INDIRECT(PTR)
Definition: postgres.h:329
#define ERROR
Definition: elog.h:46
#define VARATT_IS_SHORT(PTR)
Definition: postgres.h:339
#define VARSIZE_SHORT(PTR)
Definition: postgres.h:318
static struct varlena * toast_fetch_datum(struct varlena *attr)
Definition: detoast.c:343
#define Assert(condition)
Definition: c.h:804
static struct varlena * toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 slicelength)
Definition: detoast.c:396
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer)
Definition: postgres.h:371
static bool pg_add_s32_overflow(int32 a, int32 b, int32 *result)
Definition: int.h:104
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer)
Definition: postgres.h:373
#define VARDATA_SHORT(PTR)
Definition: postgres.h:319
void * palloc(Size size)
Definition: mcxt.c:1062
#define elog(elevel,...)
Definition: elog.h:232
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
static struct varlena * toast_decompress_datum(struct varlena *attr)
Definition: detoast.c:471
static struct varlena * toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
Definition: detoast.c:503

◆ detoast_external_attr()

struct varlena* detoast_external_attr ( struct varlena attr)

Definition at line 45 of file detoast.c.

References Assert, DatumGetEOHP(), detoast_external_attr(), EOH_flatten_into(), EOH_get_flat_size(), palloc(), varatt_indirect::pointer, PointerGetDatum, toast_fetch_datum(), VARATT_EXTERNAL_GET_POINTER, VARATT_IS_EXTERNAL, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, and VARSIZE_ANY.

Referenced by assign_simple_var(), brin_form_tuple(), detoast_attr(), detoast_attr_slice(), detoast_external_attr(), expanded_record_set_field_internal(), expanded_record_set_fields(), index_form_tuple(), make_tuple_indirect(), toast_build_flattened_tuple(), toast_flatten_tuple(), toast_tuple_init(), and tstoreReceiveSlot_detoast().

46 {
47  struct varlena *result;
48 
49  if (VARATT_IS_EXTERNAL_ONDISK(attr))
50  {
51  /*
52  * This is an external stored plain value
53  */
54  result = toast_fetch_datum(attr);
55  }
56  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
57  {
58  /*
59  * This is an indirect pointer --- dereference it
60  */
61  struct varatt_indirect redirect;
62 
63  VARATT_EXTERNAL_GET_POINTER(redirect, attr);
64  attr = (struct varlena *) redirect.pointer;
65 
66  /* nested indirect Datums aren't allowed */
68 
69  /* recurse if value is still external in some other way */
70  if (VARATT_IS_EXTERNAL(attr))
71  return detoast_external_attr(attr);
72 
73  /*
74  * Copy into the caller's memory context, in case caller tries to
75  * pfree the result.
76  */
77  result = (struct varlena *) palloc(VARSIZE_ANY(attr));
78  memcpy(result, attr, VARSIZE_ANY(attr));
79  }
80  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
81  {
82  /*
83  * This is an expanded-object pointer --- get flat format
84  */
86  Size resultsize;
87 
88  eoh = DatumGetEOHP(PointerGetDatum(attr));
89  resultsize = EOH_get_flat_size(eoh);
90  result = (struct varlena *) palloc(resultsize);
91  EOH_flatten_into(eoh, (void *) result, resultsize);
92  }
93  else
94  {
95  /*
96  * This is a plain value inside of the main tuple - why am I called?
97  */
98  result = attr;
99  }
100 
101  return result;
102 }
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: postgres.h:327
#define VARATT_IS_EXTERNAL_EXPANDED(PTR)
Definition: postgres.h:335
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
#define PointerGetDatum(X)
Definition: postgres.h:600
struct varlena * detoast_external_attr(struct varlena *attr)
Definition: detoast.c:45
#define VARATT_IS_EXTERNAL(PTR)
Definition: postgres.h:326
#define VARATT_IS_EXTERNAL_INDIRECT(PTR)
Definition: postgres.h:329
Size EOH_get_flat_size(ExpandedObjectHeader *eohptr)
Definition: expandeddatum.c:75
ExpandedObjectHeader * DatumGetEOHP(Datum d)
Definition: expandeddatum.c:29
void EOH_flatten_into(ExpandedObjectHeader *eohptr, void *result, Size allocated_size)
Definition: expandeddatum.c:81
#define VARSIZE_ANY(PTR)
Definition: postgres.h:348
static struct varlena * toast_fetch_datum(struct varlena *attr)
Definition: detoast.c:343
#define Assert(condition)
Definition: c.h:804
size_t Size
Definition: c.h:540
void * palloc(Size size)
Definition: mcxt.c:1062
Definition: c.h:621

◆ toast_datum_size()

Size toast_datum_size ( Datum  value)

Definition at line 601 of file detoast.c.

References Assert, DatumGetEOHP(), DatumGetPointer, EOH_get_flat_size(), varatt_indirect::pointer, PointerGetDatum, toast_datum_size(), VARATT_EXTERNAL_GET_EXTSIZE, VARATT_EXTERNAL_GET_POINTER, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARSIZE, and VARSIZE_SHORT.

Referenced by pg_column_size(), and toast_datum_size().

602 {
603  struct varlena *attr = (struct varlena *) DatumGetPointer(value);
604  Size result;
605 
606  if (VARATT_IS_EXTERNAL_ONDISK(attr))
607  {
608  /*
609  * Attribute is stored externally - return the extsize whether
610  * compressed or not. We do not count the size of the toast pointer
611  * ... should we?
612  */
613  struct varatt_external toast_pointer;
614 
615  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
616  result = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
617  }
618  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
619  {
620  struct varatt_indirect toast_pointer;
621 
622  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
623 
624  /* nested indirect Datums aren't allowed */
626 
627  return toast_datum_size(PointerGetDatum(toast_pointer.pointer));
628  }
629  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
630  {
632  }
633  else if (VARATT_IS_SHORT(attr))
634  {
635  result = VARSIZE_SHORT(attr);
636  }
637  else
638  {
639  /*
640  * Attribute is stored inline either compressed or not, just calculate
641  * the size of the datum in either case.
642  */
643  result = VARSIZE(attr);
644  }
645  return result;
646 }
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: postgres.h:327
static struct @142 value
#define VARATT_IS_EXTERNAL_EXPANDED(PTR)
Definition: postgres.h:335
#define VARSIZE(PTR)
Definition: postgres.h:316
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
#define PointerGetDatum(X)
Definition: postgres.h:600
Size toast_datum_size(Datum value)
Definition: detoast.c:601
#define VARATT_IS_EXTERNAL_INDIRECT(PTR)
Definition: postgres.h:329
#define VARATT_IS_SHORT(PTR)
Definition: postgres.h:339
Size EOH_get_flat_size(ExpandedObjectHeader *eohptr)
Definition: expandeddatum.c:75
#define VARSIZE_SHORT(PTR)
Definition: postgres.h:318
ExpandedObjectHeader * DatumGetEOHP(Datum d)
Definition: expandeddatum.c:29
#define Assert(condition)
Definition: c.h:804
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer)
Definition: postgres.h:371
size_t Size
Definition: c.h:540
#define DatumGetPointer(X)
Definition: postgres.h:593
Definition: c.h:621

◆ toast_decompress_datum()

static struct varlena * toast_decompress_datum ( struct varlena attr)
static

Definition at line 471 of file detoast.c.

References Assert, elog, ERROR, lz4_decompress_datum(), pglz_decompress_datum(), TOAST_COMPRESS_METHOD, TOAST_LZ4_COMPRESSION_ID, TOAST_PGLZ_COMPRESSION_ID, and VARATT_IS_COMPRESSED.

Referenced by detoast_attr(), detoast_attr_slice(), and toast_decompress_datum_slice().

472 {
473  ToastCompressionId cmid;
474 
476 
477  /*
478  * Fetch the compression method id stored in the compression header and
479  * decompress the data using the appropriate decompression routine.
480  */
481  cmid = TOAST_COMPRESS_METHOD(attr);
482  switch (cmid)
483  {
485  return pglz_decompress_datum(attr);
487  return lz4_decompress_datum(attr);
488  default:
489  elog(ERROR, "invalid compression method id %d", cmid);
490  return NULL; /* keep compiler quiet */
491  }
492 }
#define VARATT_IS_COMPRESSED(PTR)
Definition: postgres.h:325
struct varlena * pglz_decompress_datum(const struct varlena *value)
ToastCompressionId
#define ERROR
Definition: elog.h:46
#define TOAST_COMPRESS_METHOD(ptr)
#define Assert(condition)
Definition: c.h:804
#define elog(elevel,...)
Definition: elog.h:232
struct varlena * lz4_decompress_datum(const struct varlena *value)

◆ toast_decompress_datum_slice()

static struct varlena * toast_decompress_datum_slice ( struct varlena attr,
int32  slicelength 
)
static

Definition at line 503 of file detoast.c.

References Assert, elog, ERROR, lz4_decompress_datum_slice(), pglz_decompress_datum_slice(), TOAST_COMPRESS_EXTSIZE, TOAST_COMPRESS_METHOD, toast_decompress_datum(), TOAST_LZ4_COMPRESSION_ID, TOAST_PGLZ_COMPRESSION_ID, and VARATT_IS_COMPRESSED.

Referenced by detoast_attr_slice().

504 {
505  ToastCompressionId cmid;
506 
508 
509  /*
510  * Some callers may pass a slicelength that's more than the actual
511  * decompressed size. If so, just decompress normally. This avoids
512  * possibly allocating a larger-than-necessary result object, and may be
513  * faster and/or more robust as well. Notably, some versions of liblz4
514  * have been seen to give wrong results if passed an output size that is
515  * more than the data's true decompressed size.
516  */
517  if ((uint32) slicelength >= TOAST_COMPRESS_EXTSIZE(attr))
518  return toast_decompress_datum(attr);
519 
520  /*
521  * Fetch the compression method id stored in the compression header and
522  * decompress the data slice using the appropriate decompression routine.
523  */
524  cmid = TOAST_COMPRESS_METHOD(attr);
525  switch (cmid)
526  {
528  return pglz_decompress_datum_slice(attr, slicelength);
530  return lz4_decompress_datum_slice(attr, slicelength);
531  default:
532  elog(ERROR, "invalid compression method id %d", cmid);
533  return NULL; /* keep compiler quiet */
534  }
535 }
#define VARATT_IS_COMPRESSED(PTR)
Definition: postgres.h:325
ToastCompressionId
struct varlena * pglz_decompress_datum_slice(const struct varlena *value, int32 slicelength)
#define ERROR
Definition: elog.h:46
#define TOAST_COMPRESS_EXTSIZE(ptr)
unsigned int uint32
Definition: c.h:441
#define TOAST_COMPRESS_METHOD(ptr)
#define Assert(condition)
Definition: c.h:804
struct varlena * lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
#define elog(elevel,...)
Definition: elog.h:232
static struct varlena * toast_decompress_datum(struct varlena *attr)
Definition: detoast.c:471

◆ toast_fetch_datum()

static struct varlena * toast_fetch_datum ( struct varlena attr)
static

Definition at line 343 of file detoast.c.

References AccessShareLock, elog, ERROR, palloc(), SET_VARSIZE, SET_VARSIZE_COMPRESSED, table_close(), table_open(), table_relation_fetch_toast_slice(), varatt_external::va_toastrelid, varatt_external::va_valueid, VARATT_EXTERNAL_GET_EXTSIZE, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_EXTERNAL_ONDISK, and VARHDRSZ.

Referenced by detoast_attr(), detoast_attr_slice(), and detoast_external_attr().

344 {
345  Relation toastrel;
346  struct varlena *result;
347  struct varatt_external toast_pointer;
348  int32 attrsize;
349 
350  if (!VARATT_IS_EXTERNAL_ONDISK(attr))
351  elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums");
352 
353  /* Must copy to access aligned fields */
354  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
355 
356  attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
357 
358  result = (struct varlena *) palloc(attrsize + VARHDRSZ);
359 
360  if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
361  SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ);
362  else
363  SET_VARSIZE(result, attrsize + VARHDRSZ);
364 
365  if (attrsize == 0)
366  return result; /* Probably shouldn't happen, but just in
367  * case. */
368 
369  /*
370  * Open the toast relation and its indexes
371  */
372  toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
373 
374  /* Fetch all chunks */
375  table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
376  attrsize, 0, attrsize, result);
377 
378  /* Close toast table */
379  table_close(toastrel, AccessShareLock);
380 
381  return result;
382 }
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: postgres.h:327
void table_close(Relation relation, LOCKMODE lockmode)
Definition: table.c:167
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)
Definition: postgres.h:391
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
#define VARHDRSZ
Definition: c.h:627
static void table_relation_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, struct varlena *result)
Definition: tableam.h:1886
#define AccessShareLock
Definition: lockdefs.h:36
signed int int32
Definition: c.h:429
#define ERROR
Definition: elog.h:46
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer)
Definition: postgres.h:371
void * palloc(Size size)
Definition: mcxt.c:1062
#define elog(elevel,...)
Definition: elog.h:232
#define SET_VARSIZE_COMPRESSED(PTR, len)
Definition: postgres.h:344
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
Relation table_open(Oid relationId, LOCKMODE lockmode)
Definition: table.c:39

◆ toast_fetch_datum_slice()

static struct varlena * toast_fetch_datum_slice ( struct varlena attr,
int32  sliceoffset,
int32  slicelength 
)
static

Definition at line 396 of file detoast.c.

References AccessShareLock, Assert, elog, ERROR, palloc(), SET_VARSIZE, SET_VARSIZE_COMPRESSED, table_close(), table_open(), table_relation_fetch_toast_slice(), varatt_external::va_toastrelid, varatt_external::va_valueid, VARATT_EXTERNAL_GET_EXTSIZE, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_EXTERNAL_ONDISK, and VARHDRSZ.

Referenced by detoast_attr_slice().

398 {
399  Relation toastrel;
400  struct varlena *result;
401  struct varatt_external toast_pointer;
402  int32 attrsize;
403 
404  if (!VARATT_IS_EXTERNAL_ONDISK(attr))
405  elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums");
406 
407  /* Must copy to access aligned fields */
408  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
409 
410  /*
411  * It's nonsense to fetch slices of a compressed datum unless when it's a
412  * prefix -- this isn't lo_* we can't return a compressed datum which is
413  * meaningful to toast later.
414  */
415  Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset);
416 
417  attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
418 
419  if (sliceoffset >= attrsize)
420  {
421  sliceoffset = 0;
422  slicelength = 0;
423  }
424 
425  /*
426  * When fetching a prefix of a compressed external datum, account for the
427  * space required by va_tcinfo, which is stored at the beginning as an
428  * int32 value.
429  */
430  if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) && slicelength > 0)
431  slicelength = slicelength + sizeof(int32);
432 
433  /*
434  * Adjust length request if needed. (Note: our sole caller,
435  * detoast_attr_slice, protects us against sliceoffset + slicelength
436  * overflowing.)
437  */
438  if (((sliceoffset + slicelength) > attrsize) || slicelength < 0)
439  slicelength = attrsize - sliceoffset;
440 
441  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
442 
443  if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
444  SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ);
445  else
446  SET_VARSIZE(result, slicelength + VARHDRSZ);
447 
448  if (slicelength == 0)
449  return result; /* Can save a lot of work at this point! */
450 
451  /* Open the toast relation */
452  toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
453 
454  /* Fetch all chunks */
455  table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
456  attrsize, sliceoffset, slicelength,
457  result);
458 
459  /* Close toast table */
460  table_close(toastrel, AccessShareLock);
461 
462  return result;
463 }
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: postgres.h:327
void table_close(Relation relation, LOCKMODE lockmode)
Definition: table.c:167
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)
Definition: postgres.h:391
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
#define VARHDRSZ
Definition: c.h:627
static void table_relation_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, int32 sliceoffset, int32 slicelength, struct varlena *result)
Definition: tableam.h:1886
#define AccessShareLock
Definition: lockdefs.h:36
signed int int32
Definition: c.h:429
#define ERROR
Definition: elog.h:46
#define Assert(condition)
Definition: c.h:804
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer)
Definition: postgres.h:371
void * palloc(Size size)
Definition: mcxt.c:1062
#define elog(elevel,...)
Definition: elog.h:232
#define SET_VARSIZE_COMPRESSED(PTR, len)
Definition: postgres.h:344
Definition: c.h:621
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
Relation table_open(Oid relationId, LOCKMODE lockmode)
Definition: table.c:39

◆ toast_raw_datum_size()

Size toast_raw_datum_size ( Datum  value)

Definition at line 545 of file detoast.c.

References Assert, DatumGetEOHP(), DatumGetPointer, EOH_get_flat_size(), varatt_indirect::pointer, PointerGetDatum, toast_raw_datum_size(), varatt_external::va_rawsize, VARATT_EXTERNAL_GET_POINTER, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_SHORT, VARSIZE, and VARSIZE_SHORT.

Referenced by bpcharoctetlen(), build_sorted_items(), byteaeq(), byteane(), byteaoctetlen(), compute_array_stats(), compute_distinct_stats(), compute_scalar_stats(), datum_image_eq(), record_image_cmp(), text_length(), text_starts_with(), texteq(), textne(), textoctetlen(), and toast_raw_datum_size().

546 {
547  struct varlena *attr = (struct varlena *) DatumGetPointer(value);
548  Size result;
549 
550  if (VARATT_IS_EXTERNAL_ONDISK(attr))
551  {
552  /* va_rawsize is the size of the original datum -- including header */
553  struct varatt_external toast_pointer;
554 
555  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
556  result = toast_pointer.va_rawsize;
557  }
558  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
559  {
560  struct varatt_indirect toast_pointer;
561 
562  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
563 
564  /* nested indirect Datums aren't allowed */
565  Assert(!VARATT_IS_EXTERNAL_INDIRECT(toast_pointer.pointer));
566 
567  return toast_raw_datum_size(PointerGetDatum(toast_pointer.pointer));
568  }
569  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
570  {
572  }
573  else if (VARATT_IS_COMPRESSED(attr))
574  {
575  /* here, va_rawsize is just the payload size */
576  result = VARDATA_COMPRESSED_GET_EXTSIZE(attr) + VARHDRSZ;
577  }
578  else if (VARATT_IS_SHORT(attr))
579  {
580  /*
581  * we have to normalize the header length to VARHDRSZ or else the
582  * callers of this function will be confused.
583  */
584  result = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT + VARHDRSZ;
585  }
586  else
587  {
588  /* plain untoasted datum */
589  result = VARSIZE(attr);
590  }
591  return result;
592 }
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: postgres.h:327
#define VARATT_IS_COMPRESSED(PTR)
Definition: postgres.h:325
static struct @142 value
#define VARATT_IS_EXTERNAL_EXPANDED(PTR)
Definition: postgres.h:335
#define VARHDRSZ_SHORT
Definition: postgres.h:292
#define VARSIZE(PTR)
Definition: postgres.h:316
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
#define PointerGetDatum(X)
Definition: postgres.h:600
#define VARHDRSZ
Definition: c.h:627
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR)
Definition: postgres.h:365
#define VARATT_IS_EXTERNAL_INDIRECT(PTR)
Definition: postgres.h:329
#define VARATT_IS_SHORT(PTR)
Definition: postgres.h:339
Size EOH_get_flat_size(ExpandedObjectHeader *eohptr)
Definition: expandeddatum.c:75
Size toast_raw_datum_size(Datum value)
Definition: detoast.c:545
#define VARSIZE_SHORT(PTR)
Definition: postgres.h:318
ExpandedObjectHeader * DatumGetEOHP(Datum d)
Definition: expandeddatum.c:29
#define Assert(condition)
Definition: c.h:804
size_t Size
Definition: c.h:540
#define DatumGetPointer(X)
Definition: postgres.h:593
Definition: c.h:621