PostgreSQL Source Code  git master
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
detoast.h File Reference
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros

#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
 
#define TOAST_POINTER_SIZE   (VARHDRSZ_EXTERNAL + sizeof(varatt_external))
 
#define INDIRECT_POINTER_SIZE   (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
 

Functions

struct varlenadetoast_external_attr (struct varlena *attr)
 
struct varlenadetoast_attr (struct varlena *attr)
 
struct varlenadetoast_attr_slice (struct varlena *attr, int32 sliceoffset, int32 slicelength)
 
Size toast_raw_datum_size (Datum value)
 
Size toast_datum_size (Datum value)
 

Macro Definition Documentation

◆ INDIRECT_POINTER_SIZE

#define INDIRECT_POINTER_SIZE   (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))

Definition at line 34 of file detoast.h.

◆ TOAST_POINTER_SIZE

#define TOAST_POINTER_SIZE   (VARHDRSZ_EXTERNAL + sizeof(varatt_external))

Definition at line 31 of file detoast.h.

◆ VARATT_EXTERNAL_GET_POINTER

#define VARATT_EXTERNAL_GET_POINTER (   toast_pointer,
  attr 
)
Value:
do { \
varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
Assert(VARATT_IS_EXTERNAL(attre)); \
Assert(VARSIZE_EXTERNAL(attre) == sizeof(toast_pointer) + VARHDRSZ_EXTERNAL); \
memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
} while (0)
#define VARSIZE_EXTERNAL(PTR)
Definition: varatt.h:285
#define VARDATA_EXTERNAL(PTR)
Definition: varatt.h:286
#define VARATT_IS_EXTERNAL(PTR)
Definition: varatt.h:289
#define VARHDRSZ_EXTERNAL
Definition: varatt.h:253

Definition at line 22 of file detoast.h.

Function Documentation

◆ detoast_attr()

struct varlena* detoast_attr ( struct varlena attr)

Definition at line 116 of file detoast.c.

117 {
118  if (VARATT_IS_EXTERNAL_ONDISK(attr))
119  {
120  /*
121  * This is an externally stored datum --- fetch it back from there
122  */
123  attr = toast_fetch_datum(attr);
124  /* If it's compressed, decompress it */
125  if (VARATT_IS_COMPRESSED(attr))
126  {
127  struct varlena *tmp = attr;
128 
129  attr = toast_decompress_datum(tmp);
130  pfree(tmp);
131  }
132  }
133  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
134  {
135  /*
136  * This is an indirect pointer --- dereference it
137  */
138  struct varatt_indirect redirect;
139 
140  VARATT_EXTERNAL_GET_POINTER(redirect, attr);
141  attr = (struct varlena *) redirect.pointer;
142 
143  /* nested indirect Datums aren't allowed */
145 
146  /* recurse in case value is still extended in some other way */
147  attr = detoast_attr(attr);
148 
149  /* if it isn't, we'd better copy it */
150  if (attr == (struct varlena *) redirect.pointer)
151  {
152  struct varlena *result;
153 
154  result = (struct varlena *) palloc(VARSIZE_ANY(attr));
155  memcpy(result, attr, VARSIZE_ANY(attr));
156  attr = result;
157  }
158  }
159  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
160  {
161  /*
162  * This is an expanded-object pointer --- get flat format
163  */
164  attr = detoast_external_attr(attr);
165  /* flatteners are not allowed to produce compressed/short output */
166  Assert(!VARATT_IS_EXTENDED(attr));
167  }
168  else if (VARATT_IS_COMPRESSED(attr))
169  {
170  /*
171  * This is a compressed value inside of the main tuple
172  */
173  attr = toast_decompress_datum(attr);
174  }
175  else if (VARATT_IS_SHORT(attr))
176  {
177  /*
178  * This is a short-header varlena --- convert to 4-byte header format
179  */
180  Size data_size = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT;
181  Size new_size = data_size + VARHDRSZ;
182  struct varlena *new_attr;
183 
184  new_attr = (struct varlena *) palloc(new_size);
185  SET_VARSIZE(new_attr, new_size);
186  memcpy(VARDATA(new_attr), VARDATA_SHORT(attr), data_size);
187  attr = new_attr;
188  }
189 
190  return attr;
191 }
#define VARHDRSZ
Definition: c.h:695
#define Assert(condition)
Definition: c.h:861
size_t Size
Definition: c.h:608
static struct varlena * toast_fetch_datum(struct varlena *attr)
Definition: detoast.c:343
struct varlena * detoast_external_attr(struct varlena *attr)
Definition: detoast.c:45
struct varlena * detoast_attr(struct varlena *attr)
Definition: detoast.c:116
static struct varlena * toast_decompress_datum(struct varlena *attr)
Definition: detoast.c:471
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr)
Definition: detoast.h:22
void pfree(void *pointer)
Definition: mcxt.c:1521
void * palloc(Size size)
Definition: mcxt.c:1317
Definition: c.h:690
#define VARHDRSZ_SHORT
Definition: varatt.h:255
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: varatt.h:290
#define VARATT_IS_EXTERNAL_INDIRECT(PTR)
Definition: varatt.h:292
#define VARSIZE_SHORT(PTR)
Definition: varatt.h:281
#define VARATT_IS_EXTENDED(PTR)
Definition: varatt.h:303
#define VARATT_IS_EXTERNAL_EXPANDED(PTR)
Definition: varatt.h:298
#define VARATT_IS_SHORT(PTR)
Definition: varatt.h:302
#define VARSIZE_ANY(PTR)
Definition: varatt.h:311
#define VARDATA(PTR)
Definition: varatt.h:278
#define VARATT_IS_COMPRESSED(PTR)
Definition: varatt.h:288
#define SET_VARSIZE(PTR, len)
Definition: varatt.h:305
#define VARDATA_SHORT(PTR)
Definition: varatt.h:282

References Assert, detoast_external_attr(), palloc(), pfree(), varatt_indirect::pointer, SET_VARSIZE, toast_decompress_datum(), toast_fetch_datum(), VARATT_EXTERNAL_GET_POINTER, VARATT_IS_COMPRESSED, VARATT_IS_EXTENDED, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARDATA, VARDATA_SHORT, VARHDRSZ, VARHDRSZ_SHORT, VARSIZE_ANY, and VARSIZE_SHORT.

Referenced by getdatafield(), pg_detoast_datum(), pg_detoast_datum_copy(), pg_detoast_datum_packed(), toast_flatten_tuple_to_datum(), and toast_tuple_init().

◆ detoast_attr_slice()

struct varlena* detoast_attr_slice ( struct varlena attr,
int32  sliceoffset,
int32  slicelength 
)

Definition at line 205 of file detoast.c.

207 {
208  struct varlena *preslice;
209  struct varlena *result;
210  char *attrdata;
211  int32 slicelimit;
212  int32 attrsize;
213 
214  if (sliceoffset < 0)
215  elog(ERROR, "invalid sliceoffset: %d", sliceoffset);
216 
217  /*
218  * Compute slicelimit = offset + length, or -1 if we must fetch all of the
219  * value. In case of integer overflow, we must fetch all.
220  */
221  if (slicelength < 0)
222  slicelimit = -1;
223  else if (pg_add_s32_overflow(sliceoffset, slicelength, &slicelimit))
224  slicelength = slicelimit = -1;
225 
226  if (VARATT_IS_EXTERNAL_ONDISK(attr))
227  {
228  struct varatt_external toast_pointer;
229 
230  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
231 
232  /* fast path for non-compressed external datums */
233  if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
234  return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
235 
236  /*
237  * For compressed values, we need to fetch enough slices to decompress
238  * at least the requested part (when a prefix is requested).
239  * Otherwise, just fetch all slices.
240  */
241  if (slicelimit >= 0)
242  {
243  int32 max_size = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
244 
245  /*
246  * Determine maximum amount of compressed data needed for a prefix
247  * of a given length (after decompression).
248  *
249  * At least for now, if it's LZ4 data, we'll have to fetch the
250  * whole thing, because there doesn't seem to be an API call to
251  * determine how much compressed data we need to be sure of being
252  * able to decompress the required slice.
253  */
254  if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
256  max_size = pglz_maximum_compressed_size(slicelimit, max_size);
257 
258  /*
259  * Fetch enough compressed slices (compressed marker will get set
260  * automatically).
261  */
262  preslice = toast_fetch_datum_slice(attr, 0, max_size);
263  }
264  else
265  preslice = toast_fetch_datum(attr);
266  }
267  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
268  {
269  struct varatt_indirect redirect;
270 
271  VARATT_EXTERNAL_GET_POINTER(redirect, attr);
272 
273  /* nested indirect Datums aren't allowed */
274  Assert(!VARATT_IS_EXTERNAL_INDIRECT(redirect.pointer));
275 
276  return detoast_attr_slice(redirect.pointer,
277  sliceoffset, slicelength);
278  }
279  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
280  {
281  /* pass it off to detoast_external_attr to flatten */
282  preslice = detoast_external_attr(attr);
283  }
284  else
285  preslice = attr;
286 
287  Assert(!VARATT_IS_EXTERNAL(preslice));
288 
289  if (VARATT_IS_COMPRESSED(preslice))
290  {
291  struct varlena *tmp = preslice;
292 
293  /* Decompress enough to encompass the slice and the offset */
294  if (slicelimit >= 0)
295  preslice = toast_decompress_datum_slice(tmp, slicelimit);
296  else
297  preslice = toast_decompress_datum(tmp);
298 
299  if (tmp != attr)
300  pfree(tmp);
301  }
302 
303  if (VARATT_IS_SHORT(preslice))
304  {
305  attrdata = VARDATA_SHORT(preslice);
306  attrsize = VARSIZE_SHORT(preslice) - VARHDRSZ_SHORT;
307  }
308  else
309  {
310  attrdata = VARDATA(preslice);
311  attrsize = VARSIZE(preslice) - VARHDRSZ;
312  }
313 
314  /* slicing of datum for compressed cases and plain value */
315 
316  if (sliceoffset >= attrsize)
317  {
318  sliceoffset = 0;
319  slicelength = 0;
320  }
321  else if (slicelength < 0 || slicelimit > attrsize)
322  slicelength = attrsize - sliceoffset;
323 
324  result = (struct varlena *) palloc(slicelength + VARHDRSZ);
325  SET_VARSIZE(result, slicelength + VARHDRSZ);
326 
327  memcpy(VARDATA(result), attrdata + sliceoffset, slicelength);
328 
329  if (preslice != attr)
330  pfree(preslice);
331 
332  return result;
333 }
signed int int32
Definition: c.h:508
static struct varlena * toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
Definition: detoast.c:503
static struct varlena * toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 slicelength)
Definition: detoast.c:396
struct varlena * detoast_attr_slice(struct varlena *attr, int32 sliceoffset, int32 slicelength)
Definition: detoast.c:205
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:225
static bool pg_add_s32_overflow(int32 a, int32 b, int32 *result)
Definition: int.h:135
int32 pglz_maximum_compressed_size(int32 rawsize, int32 total_compressed_size)
@ TOAST_PGLZ_COMPRESSION_ID
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer)
Definition: varatt.h:336
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)
Definition: varatt.h:354
#define VARSIZE(PTR)
Definition: varatt.h:279
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer)
Definition: varatt.h:334

References Assert, detoast_external_attr(), elog, ERROR, palloc(), pfree(), pg_add_s32_overflow(), pglz_maximum_compressed_size(), varatt_indirect::pointer, SET_VARSIZE, toast_decompress_datum(), toast_decompress_datum_slice(), toast_fetch_datum(), toast_fetch_datum_slice(), TOAST_PGLZ_COMPRESSION_ID, VARATT_EXTERNAL_GET_COMPRESS_METHOD, VARATT_EXTERNAL_GET_EXTSIZE, VARATT_EXTERNAL_GET_POINTER, VARATT_EXTERNAL_IS_COMPRESSED, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARDATA, VARDATA_SHORT, VARHDRSZ, VARHDRSZ_SHORT, VARSIZE, and VARSIZE_SHORT.

Referenced by pg_detoast_datum_slice().

◆ detoast_external_attr()

struct varlena* detoast_external_attr ( struct varlena attr)

Definition at line 45 of file detoast.c.

46 {
47  struct varlena *result;
48 
49  if (VARATT_IS_EXTERNAL_ONDISK(attr))
50  {
51  /*
52  * This is an external stored plain value
53  */
54  result = toast_fetch_datum(attr);
55  }
56  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
57  {
58  /*
59  * This is an indirect pointer --- dereference it
60  */
61  struct varatt_indirect redirect;
62 
63  VARATT_EXTERNAL_GET_POINTER(redirect, attr);
64  attr = (struct varlena *) redirect.pointer;
65 
66  /* nested indirect Datums aren't allowed */
68 
69  /* recurse if value is still external in some other way */
70  if (VARATT_IS_EXTERNAL(attr))
71  return detoast_external_attr(attr);
72 
73  /*
74  * Copy into the caller's memory context, in case caller tries to
75  * pfree the result.
76  */
77  result = (struct varlena *) palloc(VARSIZE_ANY(attr));
78  memcpy(result, attr, VARSIZE_ANY(attr));
79  }
80  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
81  {
82  /*
83  * This is an expanded-object pointer --- get flat format
84  */
86  Size resultsize;
87 
88  eoh = DatumGetEOHP(PointerGetDatum(attr));
89  resultsize = EOH_get_flat_size(eoh);
90  result = (struct varlena *) palloc(resultsize);
91  EOH_flatten_into(eoh, (void *) result, resultsize);
92  }
93  else
94  {
95  /*
96  * This is a plain value inside of the main tuple - why am I called?
97  */
98  result = attr;
99  }
100 
101  return result;
102 }
ExpandedObjectHeader * DatumGetEOHP(Datum d)
Definition: expandeddatum.c:29
void EOH_flatten_into(ExpandedObjectHeader *eohptr, void *result, Size allocated_size)
Definition: expandeddatum.c:81
Size EOH_get_flat_size(ExpandedObjectHeader *eohptr)
Definition: expandeddatum.c:75
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:322

References Assert, DatumGetEOHP(), EOH_flatten_into(), EOH_get_flat_size(), palloc(), varatt_indirect::pointer, PointerGetDatum(), toast_fetch_datum(), VARATT_EXTERNAL_GET_POINTER, VARATT_IS_EXTERNAL, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, and VARSIZE_ANY.

Referenced by assign_simple_var(), brin_form_tuple(), detoast_attr(), detoast_attr_slice(), expanded_record_set_field_internal(), expanded_record_set_fields(), index_form_tuple_context(), make_tuple_indirect(), toast_build_flattened_tuple(), toast_flatten_tuple(), toast_tuple_init(), and tstoreReceiveSlot_detoast().

◆ toast_datum_size()

Size toast_datum_size ( Datum  value)

Definition at line 601 of file detoast.c.

602 {
603  struct varlena *attr = (struct varlena *) DatumGetPointer(value);
604  Size result;
605 
606  if (VARATT_IS_EXTERNAL_ONDISK(attr))
607  {
608  /*
609  * Attribute is stored externally - return the extsize whether
610  * compressed or not. We do not count the size of the toast pointer
611  * ... should we?
612  */
613  struct varatt_external toast_pointer;
614 
615  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
616  result = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
617  }
618  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
619  {
620  struct varatt_indirect toast_pointer;
621 
622  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
623 
624  /* nested indirect Datums aren't allowed */
626 
627  return toast_datum_size(PointerGetDatum(toast_pointer.pointer));
628  }
629  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
630  {
632  }
633  else if (VARATT_IS_SHORT(attr))
634  {
635  result = VARSIZE_SHORT(attr);
636  }
637  else
638  {
639  /*
640  * Attribute is stored inline either compressed or not, just calculate
641  * the size of the datum in either case.
642  */
643  result = VARSIZE(attr);
644  }
645  return result;
646 }
Size toast_datum_size(Datum value)
Definition: detoast.c:601
static struct @160 value
static Pointer DatumGetPointer(Datum X)
Definition: postgres.h:312

References Assert, DatumGetEOHP(), DatumGetPointer(), EOH_get_flat_size(), varatt_indirect::pointer, PointerGetDatum(), value, VARATT_EXTERNAL_GET_EXTSIZE, VARATT_EXTERNAL_GET_POINTER, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARSIZE, and VARSIZE_SHORT.

Referenced by pg_column_size().

◆ toast_raw_datum_size()

Size toast_raw_datum_size ( Datum  value)

Definition at line 545 of file detoast.c.

546 {
547  struct varlena *attr = (struct varlena *) DatumGetPointer(value);
548  Size result;
549 
550  if (VARATT_IS_EXTERNAL_ONDISK(attr))
551  {
552  /* va_rawsize is the size of the original datum -- including header */
553  struct varatt_external toast_pointer;
554 
555  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
556  result = toast_pointer.va_rawsize;
557  }
558  else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
559  {
560  struct varatt_indirect toast_pointer;
561 
562  VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
563 
564  /* nested indirect Datums aren't allowed */
565  Assert(!VARATT_IS_EXTERNAL_INDIRECT(toast_pointer.pointer));
566 
567  return toast_raw_datum_size(PointerGetDatum(toast_pointer.pointer));
568  }
569  else if (VARATT_IS_EXTERNAL_EXPANDED(attr))
570  {
572  }
573  else if (VARATT_IS_COMPRESSED(attr))
574  {
575  /* here, va_rawsize is just the payload size */
576  result = VARDATA_COMPRESSED_GET_EXTSIZE(attr) + VARHDRSZ;
577  }
578  else if (VARATT_IS_SHORT(attr))
579  {
580  /*
581  * we have to normalize the header length to VARHDRSZ or else the
582  * callers of this function will be confused.
583  */
584  result = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT + VARHDRSZ;
585  }
586  else
587  {
588  /* plain untoasted datum */
589  result = VARSIZE(attr);
590  }
591  return result;
592 }
Size toast_raw_datum_size(Datum value)
Definition: detoast.c:545
#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR)
Definition: varatt.h:328

References Assert, DatumGetEOHP(), DatumGetPointer(), EOH_get_flat_size(), varatt_indirect::pointer, PointerGetDatum(), varatt_external::va_rawsize, value, VARATT_EXTERNAL_GET_POINTER, VARATT_IS_COMPRESSED, VARATT_IS_EXTERNAL_EXPANDED, VARATT_IS_EXTERNAL_INDIRECT, VARATT_IS_EXTERNAL_ONDISK, VARATT_IS_SHORT, VARDATA_COMPRESSED_GET_EXTSIZE, VARHDRSZ, VARHDRSZ_SHORT, VARSIZE, and VARSIZE_SHORT.

Referenced by bpcharoctetlen(), build_sorted_items(), byteaeq(), byteane(), byteaoctetlen(), compute_array_stats(), compute_distinct_stats(), compute_scalar_stats(), datum_image_eq(), datum_image_hash(), record_image_cmp(), text_length(), text_starts_with(), texteq(), textne(), and textoctetlen().