PostgreSQL Source Code git master
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
varatt.h
Go to the documentation of this file.
1/*-------------------------------------------------------------------------
2 *
3 * varatt.h
4 * variable-length datatypes (TOAST support)
5 *
6 *
7 * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
8 * Portions Copyright (c) 1995, Regents of the University of California
9 *
10 * src/include/varatt.h
11 *
12 *-------------------------------------------------------------------------
13 */
14
15#ifndef VARATT_H
16#define VARATT_H
17
18/*
19 * struct varatt_external is a traditional "TOAST pointer", that is, the
20 * information needed to fetch a Datum stored out-of-line in a TOAST table.
21 * The data is compressed if and only if the external size stored in
22 * va_extinfo is less than va_rawsize - VARHDRSZ.
23 *
24 * This struct must not contain any padding, because we sometimes compare
25 * these pointers using memcmp.
26 *
27 * Note that this information is stored unaligned within actual tuples, so
28 * you need to memcpy from the tuple into a local struct variable before
29 * you can look at these fields! (The reason we use memcmp is to avoid
30 * having to do that just to detect equality of two TOAST pointers...)
31 */
32typedef struct varatt_external
33{
34 int32 va_rawsize; /* Original data size (includes header) */
35 uint32 va_extinfo; /* External saved size (without header) and
36 * compression method */
37 Oid va_valueid; /* Unique ID of value within TOAST table */
38 Oid va_toastrelid; /* RelID of TOAST table containing it */
40
41/*
42 * These macros define the "saved size" portion of va_extinfo. Its remaining
43 * two high-order bits identify the compression method.
44 */
45#define VARLENA_EXTSIZE_BITS 30
46#define VARLENA_EXTSIZE_MASK ((1U << VARLENA_EXTSIZE_BITS) - 1)
47
48/*
49 * struct varatt_indirect is a "TOAST pointer" representing an out-of-line
50 * Datum that's stored in memory, not in an external toast relation.
51 * The creator of such a Datum is entirely responsible that the referenced
52 * storage survives for as long as referencing pointer Datums can exist.
53 *
54 * Note that just as for struct varatt_external, this struct is stored
55 * unaligned within any containing tuple.
56 */
57typedef struct varatt_indirect
58{
59 struct varlena *pointer; /* Pointer to in-memory varlena */
61
62/*
63 * struct varatt_expanded is a "TOAST pointer" representing an out-of-line
64 * Datum that is stored in memory, in some type-specific, not necessarily
65 * physically contiguous format that is convenient for computation not
66 * storage. APIs for this, in particular the definition of struct
67 * ExpandedObjectHeader, are in src/include/utils/expandeddatum.h.
68 *
69 * Note that just as for struct varatt_external, this struct is stored
70 * unaligned within any containing tuple.
71 */
73
74typedef struct varatt_expanded
75{
78
79/*
80 * Type tag for the various sorts of "TOAST pointer" datums. The peculiar
81 * value for VARTAG_ONDISK comes from a requirement for on-disk compatibility
82 * with a previous notion that the tag field was the pointer datum's length.
83 */
84typedef enum vartag_external
85{
89 VARTAG_ONDISK = 18
91
92/* this test relies on the specific tag values above */
93#define VARTAG_IS_EXPANDED(tag) \
94 (((tag) & ~1) == VARTAG_EXPANDED_RO)
95
96#define VARTAG_SIZE(tag) \
97 ((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \
98 VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \
99 (tag) == VARTAG_ONDISK ? sizeof(varatt_external) : \
100 (AssertMacro(false), 0))
101
102/*
103 * These structs describe the header of a varlena object that may have been
104 * TOASTed. Generally, don't reference these structs directly, but use the
105 * macros below.
106 *
107 * We use separate structs for the aligned and unaligned cases because the
108 * compiler might otherwise think it could generate code that assumes
109 * alignment while touching fields of a 1-byte-header varlena.
110 */
111typedef union
112{
113 struct /* Normal varlena (4-byte length) */
114 {
117 } va_4byte;
118 struct /* Compressed-in-line format */
119 {
120 uint32 va_header;
121 uint32 va_tcinfo; /* Original data size (excludes header) and
122 * compression method; see va_extinfo */
123 char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
124 } va_compressed;
126
127typedef struct
128{
130 char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Data begins here */
132
133/* TOAST pointers are a subset of varattrib_1b with an identifying tag byte */
134typedef struct
135{
136 uint8 va_header; /* Always 0x80 or 0x01 */
137 uint8 va_tag; /* Type of datum */
138 char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Type-specific data */
140
141/*
142 * Bit layouts for varlena headers on big-endian machines:
143 *
144 * 00xxxxxx 4-byte length word, aligned, uncompressed data (up to 1G)
145 * 01xxxxxx 4-byte length word, aligned, *compressed* data (up to 1G)
146 * 10000000 1-byte length word, unaligned, TOAST pointer
147 * 1xxxxxxx 1-byte length word, unaligned, uncompressed data (up to 126b)
148 *
149 * Bit layouts for varlena headers on little-endian machines:
150 *
151 * xxxxxx00 4-byte length word, aligned, uncompressed data (up to 1G)
152 * xxxxxx10 4-byte length word, aligned, *compressed* data (up to 1G)
153 * 00000001 1-byte length word, unaligned, TOAST pointer
154 * xxxxxxx1 1-byte length word, unaligned, uncompressed data (up to 126b)
155 *
156 * The "xxx" bits are the length field (which includes itself in all cases).
157 * In the big-endian case we mask to extract the length, in the little-endian
158 * case we shift. Note that in both cases the flag bits are in the physically
159 * first byte. Also, it is not possible for a 1-byte length word to be zero;
160 * this lets us disambiguate alignment padding bytes from the start of an
161 * unaligned datum. (We now *require* pad bytes to be filled with zero!)
162 *
163 * In TOAST pointers the va_tag field (see varattrib_1b_e) is used to discern
164 * the specific type and length of the pointer datum.
165 */
166
167/*
168 * Endian-dependent macros. These are considered internal --- use the
169 * external macros below instead of using these directly.
170 *
171 * Note: IS_1B is true for external toast records but VARSIZE_1B will return 0
172 * for such records. Hence you should usually check for IS_EXTERNAL before
173 * checking for IS_1B.
174 */
175
176#ifdef WORDS_BIGENDIAN
177
178#define VARATT_IS_4B(PTR) \
179 ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00)
180#define VARATT_IS_4B_U(PTR) \
181 ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x00)
182#define VARATT_IS_4B_C(PTR) \
183 ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x40)
184#define VARATT_IS_1B(PTR) \
185 ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x80)
186#define VARATT_IS_1B_E(PTR) \
187 ((((varattrib_1b *) (PTR))->va_header) == 0x80)
188#define VARATT_NOT_PAD_BYTE(PTR) \
189 (*((uint8 *) (PTR)) != 0)
190
191/* VARSIZE_4B() should only be used on known-aligned data */
192#define VARSIZE_4B(PTR) \
193 (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF)
194#define VARSIZE_1B(PTR) \
195 (((varattrib_1b *) (PTR))->va_header & 0x7F)
196#define VARTAG_1B_E(PTR) \
197 (((varattrib_1b_e *) (PTR))->va_tag)
198
199#define SET_VARSIZE_4B(PTR,len) \
200 (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF)
201#define SET_VARSIZE_4B_C(PTR,len) \
202 (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000)
203#define SET_VARSIZE_1B(PTR,len) \
204 (((varattrib_1b *) (PTR))->va_header = (len) | 0x80)
205#define SET_VARTAG_1B_E(PTR,tag) \
206 (((varattrib_1b_e *) (PTR))->va_header = 0x80, \
207 ((varattrib_1b_e *) (PTR))->va_tag = (tag))
208
209#else /* !WORDS_BIGENDIAN */
210
211#define VARATT_IS_4B(PTR) \
212 ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00)
213#define VARATT_IS_4B_U(PTR) \
214 ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x00)
215#define VARATT_IS_4B_C(PTR) \
216 ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x02)
217#define VARATT_IS_1B(PTR) \
218 ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x01)
219#define VARATT_IS_1B_E(PTR) \
220 ((((varattrib_1b *) (PTR))->va_header) == 0x01)
221#define VARATT_NOT_PAD_BYTE(PTR) \
222 (*((uint8 *) (PTR)) != 0)
223
224/* VARSIZE_4B() should only be used on known-aligned data */
225#define VARSIZE_4B(PTR) \
226 ((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF)
227#define VARSIZE_1B(PTR) \
228 ((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F)
229#define VARTAG_1B_E(PTR) \
230 (((varattrib_1b_e *) (PTR))->va_tag)
231
232#define SET_VARSIZE_4B(PTR,len) \
233 (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2))
234#define SET_VARSIZE_4B_C(PTR,len) \
235 (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02)
236#define SET_VARSIZE_1B(PTR,len) \
237 (((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01)
238#define SET_VARTAG_1B_E(PTR,tag) \
239 (((varattrib_1b_e *) (PTR))->va_header = 0x01, \
240 ((varattrib_1b_e *) (PTR))->va_tag = (tag))
241
242#endif /* WORDS_BIGENDIAN */
243
244#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
245#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
246#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
247#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
248
249/*
250 * Externally visible TOAST macros begin here.
251 */
252
253#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
254#define VARHDRSZ_COMPRESSED offsetof(varattrib_4b, va_compressed.va_data)
255#define VARHDRSZ_SHORT offsetof(varattrib_1b, va_data)
256
257#define VARATT_SHORT_MAX 0x7F
258#define VARATT_CAN_MAKE_SHORT(PTR) \
259 (VARATT_IS_4B_U(PTR) && \
260 (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) <= VARATT_SHORT_MAX)
261#define VARATT_CONVERTED_SHORT_SIZE(PTR) \
262 (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT)
263
264/*
265 * In consumers oblivious to data alignment, call PG_DETOAST_DATUM_PACKED(),
266 * VARDATA_ANY(), VARSIZE_ANY() and VARSIZE_ANY_EXHDR(). Elsewhere, call
267 * PG_DETOAST_DATUM(), VARDATA() and VARSIZE(). Directly fetching an int16,
268 * int32 or wider field in the struct representing the datum layout requires
269 * aligned data. memcpy() is alignment-oblivious, as are most operations on
270 * datatypes, such as text, whose layout struct contains only char fields.
271 *
272 * Code assembling a new datum should call VARDATA() and SET_VARSIZE().
273 * (Datums begin life untoasted.)
274 *
275 * Other macros here should usually be used only by tuple assembly/disassembly
276 * code and code that specifically wants to work with still-toasted Datums.
277 */
278#define VARDATA(PTR) VARDATA_4B(PTR)
279#define VARSIZE(PTR) VARSIZE_4B(PTR)
280
281#define VARSIZE_SHORT(PTR) VARSIZE_1B(PTR)
282#define VARDATA_SHORT(PTR) VARDATA_1B(PTR)
283
284#define VARTAG_EXTERNAL(PTR) VARTAG_1B_E(PTR)
285#define VARSIZE_EXTERNAL(PTR) (VARHDRSZ_EXTERNAL + VARTAG_SIZE(VARTAG_EXTERNAL(PTR)))
286#define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR)
287
288#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR)
289#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR)
290#define VARATT_IS_EXTERNAL_ONDISK(PTR) \
291 (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK)
292#define VARATT_IS_EXTERNAL_INDIRECT(PTR) \
293 (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT)
294#define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \
295 (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RO)
296#define VARATT_IS_EXTERNAL_EXPANDED_RW(PTR) \
297 (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_EXPANDED_RW)
298#define VARATT_IS_EXTERNAL_EXPANDED(PTR) \
299 (VARATT_IS_EXTERNAL(PTR) && VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
300#define VARATT_IS_EXTERNAL_NON_EXPANDED(PTR) \
301 (VARATT_IS_EXTERNAL(PTR) && !VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
302#define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR)
303#define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR))
304
305#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len)
306#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len)
307#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len)
308
309#define SET_VARTAG_EXTERNAL(PTR, tag) SET_VARTAG_1B_E(PTR, tag)
310
311#define VARSIZE_ANY(PTR) \
312 (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR) : \
313 (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \
314 VARSIZE_4B(PTR)))
315
316/* Size of a varlena data, excluding header */
317#define VARSIZE_ANY_EXHDR(PTR) \
318 (VARATT_IS_1B_E(PTR) ? VARSIZE_EXTERNAL(PTR)-VARHDRSZ_EXTERNAL : \
319 (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-VARHDRSZ_SHORT : \
320 VARSIZE_4B(PTR)-VARHDRSZ))
321
322/* caution: this will not work on an external or compressed-in-line Datum */
323/* caution: this will return a possibly unaligned pointer */
324#define VARDATA_ANY(PTR) \
325 (VARATT_IS_1B(PTR) ? VARDATA_1B(PTR) : VARDATA_4B(PTR))
326
327/* Decompressed size and compression method of a compressed-in-line Datum */
328#define VARDATA_COMPRESSED_GET_EXTSIZE(PTR) \
329 (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_EXTSIZE_MASK)
330#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
331 (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS)
332
333/* Same for external Datums; but note argument is a struct varatt_external */
334#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
335 ((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK)
336#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \
337 ((toast_pointer).va_extinfo >> VARLENA_EXTSIZE_BITS)
338
339#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
340 do { \
341 Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
342 (cm) == TOAST_LZ4_COMPRESSION_ID); \
343 ((toast_pointer).va_extinfo = \
344 (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
345 } while (0)
346
347/*
348 * Testing whether an externally-stored value is compressed now requires
349 * comparing size stored in va_extinfo (the actual length of the external data)
350 * to rawsize (the original uncompressed datum's size). The latter includes
351 * VARHDRSZ overhead, the former doesn't. We never use compression unless it
352 * actually saves space, so we expect either equality or less-than.
353 */
354#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
355 (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \
356 (toast_pointer).va_rawsize - VARHDRSZ)
357
358#endif
uint8_t uint8
Definition: c.h:483
#define FLEXIBLE_ARRAY_MEMBER
Definition: c.h:417
int32_t int32
Definition: c.h:481
uint32_t uint32
Definition: c.h:485
unsigned int Oid
Definition: postgres_ext.h:31
ExpandedObjectHeader * eohptr
Definition: varatt.h:76
uint32 va_extinfo
Definition: varatt.h:35
int32 va_rawsize
Definition: varatt.h:34
Oid va_valueid
Definition: varatt.h:37
Oid va_toastrelid
Definition: varatt.h:38
struct varlena * pointer
Definition: varatt.h:59
uint8 va_tag
Definition: varatt.h:137
uint8 va_header
Definition: varatt.h:136
uint8 va_header
Definition: varatt.h:129
Definition: c.h:641
uint32 va_header
Definition: varatt.h:115
uint32 va_tcinfo
Definition: varatt.h:121
struct varatt_external varatt_external
struct varatt_indirect varatt_indirect
struct varatt_expanded varatt_expanded
vartag_external
Definition: varatt.h:85
@ VARTAG_ONDISK
Definition: varatt.h:89
@ VARTAG_EXPANDED_RW
Definition: varatt.h:88
@ VARTAG_INDIRECT
Definition: varatt.h:86
@ VARTAG_EXPANDED_RO
Definition: varatt.h:87