PostgreSQL Source Code git master
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
toast_helper.c
Go to the documentation of this file.
1/*-------------------------------------------------------------------------
2 *
3 * toast_helper.c
4 * Helper functions for table AMs implementing compressed or
5 * out-of-line storage of varlena attributes.
6 *
7 * Copyright (c) 2000-2025, PostgreSQL Global Development Group
8 *
9 * IDENTIFICATION
10 * src/backend/access/table/toast_helper.c
11 *
12 *-------------------------------------------------------------------------
13 */
14
15#include "postgres.h"
16
17#include "access/detoast.h"
18#include "access/toast_helper.h"
20#include "catalog/pg_type_d.h"
21#include "varatt.h"
22
23
24/*
25 * Prepare to TOAST a tuple.
26 *
27 * tupleDesc, toast_values, and toast_isnull are required parameters; they
28 * provide the necessary details about the tuple to be toasted.
29 *
30 * toast_oldvalues and toast_oldisnull should be NULL for a newly-inserted
31 * tuple; for an update, they should describe the existing tuple.
32 *
33 * All of these arrays should have a length equal to tupleDesc->natts.
34 *
35 * On return, toast_flags and toast_attr will have been initialized.
36 * toast_flags is just a single uint8, but toast_attr is a caller-provided
37 * array with a length equal to tupleDesc->natts. The caller need not
38 * perform any initialization of the array before calling this function.
39 */
40void
42{
43 TupleDesc tupleDesc = ttc->ttc_rel->rd_att;
44 int numAttrs = tupleDesc->natts;
45 int i;
46
47 ttc->ttc_flags = 0;
48
49 for (i = 0; i < numAttrs; i++)
50 {
51 Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
52 struct varlena *old_value;
53 struct varlena *new_value;
54
55 ttc->ttc_attr[i].tai_colflags = 0;
56 ttc->ttc_attr[i].tai_oldexternal = NULL;
57 ttc->ttc_attr[i].tai_compression = att->attcompression;
58
59 if (ttc->ttc_oldvalues != NULL)
60 {
61 /*
62 * For UPDATE get the old and new values of this attribute
63 */
64 old_value =
65 (struct varlena *) DatumGetPointer(ttc->ttc_oldvalues[i]);
66 new_value =
67 (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
68
69 /*
70 * If the old value is stored on disk, check if it has changed so
71 * we have to delete it later.
72 */
73 if (att->attlen == -1 && !ttc->ttc_oldisnull[i] &&
75 {
76 if (ttc->ttc_isnull[i] ||
77 !VARATT_IS_EXTERNAL_ONDISK(new_value) ||
78 memcmp((char *) old_value, (char *) new_value,
79 VARSIZE_EXTERNAL(old_value)) != 0)
80 {
81 /*
82 * The old external stored value isn't needed any more
83 * after the update
84 */
87 }
88 else
89 {
90 /*
91 * This attribute isn't changed by this update so we reuse
92 * the original reference to the old value in the new
93 * tuple.
94 */
96 continue;
97 }
98 }
99 }
100 else
101 {
102 /*
103 * For INSERT simply get the new value
104 */
105 new_value = (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
106 }
107
108 /*
109 * Handle NULL attributes
110 */
111 if (ttc->ttc_isnull[i])
112 {
115 continue;
116 }
117
118 /*
119 * Now look at varlena attributes
120 */
121 if (att->attlen == -1)
122 {
123 /*
124 * If the table's attribute says PLAIN always, force it so.
125 */
126 if (att->attstorage == TYPSTORAGE_PLAIN)
128
129 /*
130 * We took care of UPDATE above, so any external value we find
131 * still in the tuple must be someone else's that we cannot reuse
132 * (this includes the case of an out-of-line in-memory datum).
133 * Fetch it back (without decompression, unless we are forcing
134 * PLAIN storage). If necessary, we'll push it out as a new
135 * external value below.
136 */
137 if (VARATT_IS_EXTERNAL(new_value))
138 {
139 ttc->ttc_attr[i].tai_oldexternal = new_value;
140 if (att->attstorage == TYPSTORAGE_PLAIN)
141 new_value = detoast_attr(new_value);
142 else
143 new_value = detoast_external_attr(new_value);
144 ttc->ttc_values[i] = PointerGetDatum(new_value);
147 }
148
149 /*
150 * Remember the size of this attribute
151 */
152 ttc->ttc_attr[i].tai_size = VARSIZE_ANY(new_value);
153 }
154 else
155 {
156 /*
157 * Not a varlena attribute, plain storage always
158 */
160 }
161 }
162}
163
164/*
165 * Find the largest varlena attribute that satisfies certain criteria.
166 *
167 * The relevant column must not be marked TOASTCOL_IGNORE, and if the
168 * for_compression flag is passed as true, it must also not be marked
169 * TOASTCOL_INCOMPRESSIBLE.
170 *
171 * The column must have attstorage EXTERNAL or EXTENDED if check_main is
172 * false, and must have attstorage MAIN if check_main is true.
173 *
174 * The column must have a minimum size of MAXALIGN(TOAST_POINTER_SIZE);
175 * if not, no benefit is to be expected by compressing it.
176 *
177 * The return value is the index of the biggest suitable column, or
178 * -1 if there is none.
179 */
180int
182 bool for_compression, bool check_main)
183{
184 TupleDesc tupleDesc = ttc->ttc_rel->rd_att;
185 int numAttrs = tupleDesc->natts;
186 int biggest_attno = -1;
187 int32 biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
188 int32 skip_colflags = TOASTCOL_IGNORE;
189 int i;
190
191 if (for_compression)
192 skip_colflags |= TOASTCOL_INCOMPRESSIBLE;
193
194 for (i = 0; i < numAttrs; i++)
195 {
196 Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
197
198 if ((ttc->ttc_attr[i].tai_colflags & skip_colflags) != 0)
199 continue;
201 continue; /* can't happen, toast_action would be PLAIN */
202 if (for_compression &&
204 continue;
205 if (check_main && att->attstorage != TYPSTORAGE_MAIN)
206 continue;
207 if (!check_main && att->attstorage != TYPSTORAGE_EXTENDED &&
208 att->attstorage != TYPSTORAGE_EXTERNAL)
209 continue;
210
211 if (ttc->ttc_attr[i].tai_size > biggest_size)
212 {
213 biggest_attno = i;
214 biggest_size = ttc->ttc_attr[i].tai_size;
215 }
216 }
217
218 return biggest_attno;
219}
220
221/*
222 * Try compression for an attribute.
223 *
224 * If we find that the attribute is not compressible, mark it so.
225 */
226void
228{
229 Datum *value = &ttc->ttc_values[attribute];
230 Datum new_value;
231 ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
232
233 new_value = toast_compress_datum(*value, attr->tai_compression);
234
235 if (DatumGetPointer(new_value) != NULL)
236 {
237 /* successful compression */
238 if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
240 *value = new_value;
244 }
245 else
246 {
247 /* incompressible, ignore on subsequent compression passes */
249 }
250}
251
252/*
253 * Move an attribute to external storage.
254 */
255void
257{
258 Datum *value = &ttc->ttc_values[attribute];
259 Datum old_value = *value;
260 ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
261
263 *value = toast_save_datum(ttc->ttc_rel, old_value, attr->tai_oldexternal,
264 options);
265 if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
266 pfree(DatumGetPointer(old_value));
269}
270
271/*
272 * Perform appropriate cleanup after one tuple has been subjected to TOAST.
273 */
274void
276{
277 TupleDesc tupleDesc = ttc->ttc_rel->rd_att;
278 int numAttrs = tupleDesc->natts;
279
280 /*
281 * Free allocated temp values
282 */
283 if ((ttc->ttc_flags & TOAST_NEEDS_FREE) != 0)
284 {
285 int i;
286
287 for (i = 0; i < numAttrs; i++)
288 {
289 ToastAttrInfo *attr = &ttc->ttc_attr[i];
290
291 if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
293 }
294 }
295
296 /*
297 * Delete external values from the old tuple
298 */
299 if ((ttc->ttc_flags & TOAST_NEEDS_DELETE_OLD) != 0)
300 {
301 int i;
302
303 for (i = 0; i < numAttrs; i++)
304 {
305 ToastAttrInfo *attr = &ttc->ttc_attr[i];
306
307 if ((attr->tai_colflags & TOASTCOL_NEEDS_DELETE_OLD) != 0)
308 toast_delete_datum(ttc->ttc_rel, ttc->ttc_oldvalues[i], false);
309 }
310 }
311}
312
313/*
314 * Check for external stored attributes and delete them from the secondary
315 * relation.
316 */
317void
318toast_delete_external(Relation rel, const Datum *values, const bool *isnull,
319 bool is_speculative)
320{
321 TupleDesc tupleDesc = rel->rd_att;
322 int numAttrs = tupleDesc->natts;
323 int i;
324
325 for (i = 0; i < numAttrs; i++)
326 {
327 if (TupleDescCompactAttr(tupleDesc, i)->attlen == -1)
328 {
329 Datum value = values[i];
330
331 if (isnull[i])
332 continue;
334 toast_delete_datum(rel, value, is_speculative);
335 }
336 }
337}
static Datum values[MAXATTR]
Definition: bootstrap.c:151
#define MAXALIGN(LEN)
Definition: c.h:765
int32_t int32
Definition: c.h:481
struct varlena * detoast_attr(struct varlena *attr)
Definition: detoast.c:116
struct varlena * detoast_external_attr(struct varlena *attr)
Definition: detoast.c:45
#define TOAST_POINTER_SIZE
Definition: detoast.h:31
static struct @161 value
int i
Definition: isn.c:72
void pfree(void *pointer)
Definition: mcxt.c:1521
int16 attlen
Definition: pg_attribute.h:59
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:200
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:322
uintptr_t Datum
Definition: postgres.h:64
static Pointer DatumGetPointer(Datum X)
Definition: postgres.h:312
TupleDesc rd_att
Definition: rel.h:112
uint8 tai_colflags
Definition: toast_helper.h:34
struct varlena * tai_oldexternal
Definition: toast_helper.h:32
char tai_compression
Definition: toast_helper.h:35
ToastAttrInfo * ttc_attr
Definition: toast_helper.h:62
Datum * ttc_oldvalues
Definition: toast_helper.h:52
Definition: c.h:641
void toast_tuple_init(ToastTupleContext *ttc)
Definition: toast_helper.c:41
void toast_delete_external(Relation rel, const Datum *values, const bool *isnull, bool is_speculative)
Definition: toast_helper.c:318
void toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
Definition: toast_helper.c:227
void toast_tuple_externalize(ToastTupleContext *ttc, int attribute, int options)
Definition: toast_helper.c:256
void toast_tuple_cleanup(ToastTupleContext *ttc)
Definition: toast_helper.c:275
int toast_tuple_find_biggest_attribute(ToastTupleContext *ttc, bool for_compression, bool check_main)
Definition: toast_helper.c:181
#define TOAST_NEEDS_CHANGE
Definition: toast_helper.h:81
#define TOAST_HAS_NULLS
Definition: toast_helper.h:80
#define TOASTCOL_NEEDS_DELETE_OLD
Definition: toast_helper.h:99
#define TOAST_NEEDS_FREE
Definition: toast_helper.h:79
#define TOASTCOL_INCOMPRESSIBLE
Definition: toast_helper.h:102
#define TOASTCOL_IGNORE
Definition: toast_helper.h:101
#define TOAST_NEEDS_DELETE_OLD
Definition: toast_helper.h:78
#define TOASTCOL_NEEDS_FREE
Definition: toast_helper.h:100
void toast_delete_datum(Relation rel, Datum value, bool is_speculative)
Datum toast_save_datum(Relation rel, Datum value, struct varlena *oldexternal, int options)
Datum toast_compress_datum(Datum value, char cmethod)
static FormData_pg_attribute * TupleDescAttr(TupleDesc tupdesc, int i)
Definition: tupdesc.h:152
static CompactAttribute * TupleDescCompactAttr(TupleDesc tupdesc, int i)
Definition: tupdesc.h:169
#define VARATT_IS_EXTERNAL_ONDISK(PTR)
Definition: varatt.h:290
#define VARSIZE_ANY(PTR)
Definition: varatt.h:311
#define VARATT_IS_COMPRESSED(PTR)
Definition: varatt.h:288
#define VARSIZE_EXTERNAL(PTR)
Definition: varatt.h:285
#define VARSIZE(PTR)
Definition: varatt.h:279
#define VARATT_IS_EXTERNAL(PTR)
Definition: varatt.h:289