PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
tuptoaster.h
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * tuptoaster.h
4  * POSTGRES definitions for external and compressed storage
5  * of variable size attributes.
6  *
7  * Copyright (c) 2000-2017, PostgreSQL Global Development Group
8  *
9  * src/include/access/tuptoaster.h
10  *
11  *-------------------------------------------------------------------------
12  */
13 #ifndef TUPTOASTER_H
14 #define TUPTOASTER_H
15 
16 #include "access/htup_details.h"
17 #include "storage/lockdefs.h"
18 #include "utils/relcache.h"
19 
20 /*
21  * This enables de-toasting of index entries. Needed until VACUUM is
22  * smart enough to rebuild indexes from scratch.
23  */
24 #define TOAST_INDEX_HACK
25 
26 
27 /*
28  * Find the maximum size of a tuple if there are to be N tuples per page.
29  */
30 #define MaximumBytesPerTuple(tuplesPerPage) \
31  MAXALIGN_DOWN((BLCKSZ - \
32  MAXALIGN(SizeOfPageHeaderData + (tuplesPerPage) * sizeof(ItemIdData))) \
33  / (tuplesPerPage))
34 
35 /*
36  * These symbols control toaster activation. If a tuple is larger than
37  * TOAST_TUPLE_THRESHOLD, we will try to toast it down to no more than
38  * TOAST_TUPLE_TARGET bytes through compressing compressible fields and
39  * moving EXTENDED and EXTERNAL data out-of-line.
40  *
41  * The numbers need not be the same, though they currently are. It doesn't
42  * make sense for TARGET to exceed THRESHOLD, but it could be useful to make
43  * it be smaller.
44  *
45  * Currently we choose both values to match the largest tuple size for which
46  * TOAST_TUPLES_PER_PAGE tuples can fit on a heap page.
47  *
48  * XXX while these can be modified without initdb, some thought needs to be
49  * given to needs_toast_table() in toasting.c before unleashing random
50  * changes. Also see LOBLKSIZE in large_object.h, which can *not* be
51  * changed without initdb.
52  */
53 #define TOAST_TUPLES_PER_PAGE 4
54 
55 #define TOAST_TUPLE_THRESHOLD MaximumBytesPerTuple(TOAST_TUPLES_PER_PAGE)
56 
57 #define TOAST_TUPLE_TARGET TOAST_TUPLE_THRESHOLD
58 
59 /*
60  * The code will also consider moving MAIN data out-of-line, but only as a
61  * last resort if the previous steps haven't reached the target tuple size.
62  * In this phase we use a different target size, currently equal to the
63  * largest tuple that will fit on a heap page. This is reasonable since
64  * the user has told us to keep the data in-line if at all possible.
65  */
66 #define TOAST_TUPLES_PER_PAGE_MAIN 1
67 
68 #define TOAST_TUPLE_TARGET_MAIN MaximumBytesPerTuple(TOAST_TUPLES_PER_PAGE_MAIN)
69 
70 /*
71  * If an index value is larger than TOAST_INDEX_TARGET, we will try to
72  * compress it (we can't move it out-of-line, however). Note that this
73  * number is per-datum, not per-tuple, for simplicity in index_form_tuple().
74  */
75 #define TOAST_INDEX_TARGET (MaxHeapTupleSize / 16)
76 
77 /*
78  * When we store an oversize datum externally, we divide it into chunks
79  * containing at most TOAST_MAX_CHUNK_SIZE data bytes. This number *must*
80  * be small enough that the completed toast-table tuple (including the
81  * ID and sequence fields and all overhead) will fit on a page.
82  * The coding here sets the size on the theory that we want to fit
83  * EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page.
84  *
85  * NB: Changing TOAST_MAX_CHUNK_SIZE requires an initdb.
86  */
87 #define EXTERN_TUPLES_PER_PAGE 4 /* tweak only this */
88 
89 #define EXTERN_TUPLE_MAX_SIZE MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE)
90 
91 #define TOAST_MAX_CHUNK_SIZE \
92  (EXTERN_TUPLE_MAX_SIZE - \
93  MAXALIGN(SizeofHeapTupleHeader) - \
94  sizeof(Oid) - \
95  sizeof(int32) - \
96  VARHDRSZ)
97 
98 /* Size of an EXTERNAL datum that contains a standard TOAST pointer */
99 #define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external))
100 
101 /* Size of an EXTERNAL datum that contains an indirection pointer */
102 #define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
103 
104 /*
105  * Testing whether an externally-stored value is compressed now requires
106  * comparing extsize (the actual length of the external data) to rawsize
107  * (the original uncompressed datum's size). The latter includes VARHDRSZ
108  * overhead, the former doesn't. We never use compression unless it actually
109  * saves space, so we expect either equality or less-than.
110  */
111 #define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
112  ((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ)
113 
114 /*
115  * Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
116  * into a local "struct varatt_external" toast pointer. This should be
117  * just a memcpy, but some versions of gcc seem to produce broken code
118  * that assumes the datum contents are aligned. Introducing an explicit
119  * intermediate "varattrib_1b_e *" variable seems to fix it.
120  */
121 #define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \
122 do { \
123  varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
124  Assert(VARATT_IS_EXTERNAL(attre)); \
125  Assert(VARSIZE_EXTERNAL(attre) == sizeof(toast_pointer) + VARHDRSZ_EXTERNAL); \
126  memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
127 } while (0)
128 
129 /* ----------
130  * toast_insert_or_update -
131  *
132  * Called by heap_insert() and heap_update().
133  * ----------
134  */
136  HeapTuple newtup, HeapTuple oldtup,
137  int options);
138 
139 /* ----------
140  * toast_delete -
141  *
142  * Called by heap_delete().
143  * ----------
144  */
145 extern void toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative);
146 
147 /* ----------
148  * heap_tuple_fetch_attr() -
149  *
150  * Fetches an external stored attribute from the toast
151  * relation. Does NOT decompress it, if stored external
152  * in compressed format.
153  * ----------
154  */
155 extern struct varlena *heap_tuple_fetch_attr(struct varlena *attr);
156 
157 /* ----------
158  * heap_tuple_untoast_attr() -
159  *
160  * Fully detoasts one attribute, fetching and/or decompressing
161  * it as needed.
162  * ----------
163  */
164 extern struct varlena *heap_tuple_untoast_attr(struct varlena *attr);
165 
166 /* ----------
167  * heap_tuple_untoast_attr_slice() -
168  *
169  * Fetches only the specified portion of an attribute.
170  * (Handles all cases for attribute storage)
171  * ----------
172  */
173 extern struct varlena *heap_tuple_untoast_attr_slice(struct varlena *attr,
174  int32 sliceoffset,
175  int32 slicelength);
176 
177 /* ----------
178  * toast_flatten_tuple -
179  *
180  * "Flatten" a tuple to contain no out-of-line toasted fields.
181  * (This does not eliminate compressed or short-header datums.)
182  * ----------
183  */
185 
186 /* ----------
187  * toast_flatten_tuple_to_datum -
188  *
189  * "Flatten" a tuple containing out-of-line toasted fields into a Datum.
190  * ----------
191  */
193  uint32 tup_len,
195 
196 /* ----------
197  * toast_build_flattened_tuple -
198  *
199  * Build a tuple containing no out-of-line toasted fields.
200  * (This does not eliminate compressed or short-header datums.)
201  * ----------
202  */
204  Datum *values,
205  bool *isnull);
206 
207 /* ----------
208  * toast_compress_datum -
209  *
210  * Create a compressed version of a varlena datum, if possible
211  * ----------
212  */
214 
215 /* ----------
216  * toast_raw_datum_size -
217  *
218  * Return the raw (detoasted) size of a varlena datum
219  * ----------
220  */
222 
223 /* ----------
224  * toast_datum_size -
225  *
226  * Return the storage size of a varlena datum
227  * ----------
228  */
230 
231 /* ----------
232  * toast_get_valid_index -
233  *
234  * Return OID of valid index associated to a toast relation
235  * ----------
236  */
237 extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock);
238 
239 #endif /* TUPTOASTER_H */
int LOCKMODE
Definition: lockdefs.h:26
Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock)
Definition: tuptoaster.c:1431
unsigned int Oid
Definition: postgres_ext.h:31
signed int int32
Definition: c.h:256
Size toast_datum_size(Datum value)
Definition: tuptoaster.c:409
static struct @121 value
Datum toast_compress_datum(Datum value)
Definition: tuptoaster.c:1373
unsigned int uint32
Definition: c.h:268
Datum toast_flatten_tuple_to_datum(HeapTupleHeader tup, uint32 tup_len, TupleDesc tupleDesc)
Definition: tuptoaster.c:1187
Size toast_raw_datum_size(Datum value)
Definition: tuptoaster.c:353
uintptr_t Datum
Definition: postgres.h:372
HeapTuple toast_flatten_tuple(HeapTuple tup, TupleDesc tupleDesc)
Definition: tuptoaster.c:1084
struct varlena * heap_tuple_untoast_attr_slice(struct varlena *attr, int32 sliceoffset, int32 slicelength)
Definition: tuptoaster.c:258
void toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative)
Definition: tuptoaster.c:464
struct varlena * heap_tuple_untoast_attr(struct varlena *attr)
Definition: tuptoaster.c:172
size_t Size
Definition: c.h:356
HeapTuple toast_build_flattened_tuple(TupleDesc tupleDesc, Datum *values, bool *isnull)
Definition: tuptoaster.c:1305
static Datum values[MAXATTR]
Definition: bootstrap.c:163
Definition: c.h:439
HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, int options)
Definition: tuptoaster.c:536
struct varlena * heap_tuple_fetch_attr(struct varlena *attr)
Definition: tuptoaster.c:101