PostgreSQL Source Code  git master
nbtdedup.c File Reference
#include "postgres.h"
#include "access/nbtree.h"
#include "access/nbtxlog.h"
#include "access/xloginsert.h"
#include "miscadmin.h"
#include "utils/rel.h"
Include dependency graph for nbtdedup.c:

Go to the source code of this file.

Functions

static void _bt_bottomupdel_finish_pending (Page page, BTDedupState state, TM_IndexDeleteOp *delstate)
 
static bool _bt_do_singleval (Relation rel, Page page, BTDedupState state, OffsetNumber minoff, IndexTuple newitem)
 
static void _bt_singleval_fillfactor (Page page, BTDedupState state, Size newitemsz)
 
void _bt_dedup_pass (Relation rel, Buffer buf, IndexTuple newitem, Size newitemsz, bool bottomupdedup)
 
bool _bt_bottomupdel_pass (Relation rel, Buffer buf, Relation heapRel, Size newitemsz)
 
void _bt_dedup_start_pending (BTDedupState state, IndexTuple base, OffsetNumber baseoff)
 
bool _bt_dedup_save_htid (BTDedupState state, IndexTuple itup)
 
Size _bt_dedup_finish_pending (Page newpage, BTDedupState state)
 
IndexTuple _bt_form_posting (IndexTuple base, ItemPointer htids, int nhtids)
 
void _bt_update_posting (BTVacuumPosting vacposting)
 
IndexTuple _bt_swap_posting (IndexTuple newitem, IndexTuple oposting, int postingoff)
 

Function Documentation

◆ _bt_bottomupdel_finish_pending()

static void _bt_bottomupdel_finish_pending ( Page  page,
BTDedupState  state,
TM_IndexDeleteOp delstate 
)
static

Definition at line 648 of file nbtdedup.c.

650 {
651  bool dupinterval = (state->nitems > 1);
652 
653  Assert(state->nitems > 0);
654  Assert(state->nitems <= state->nhtids);
655  Assert(state->intervals[state->nintervals].baseoff == state->baseoff);
656 
657  for (int i = 0; i < state->nitems; i++)
658  {
659  OffsetNumber offnum = state->baseoff + i;
660  ItemId itemid = PageGetItemId(page, offnum);
661  IndexTuple itup = (IndexTuple) PageGetItem(page, itemid);
662  TM_IndexDelete *ideltid = &delstate->deltids[delstate->ndeltids];
663  TM_IndexStatus *istatus = &delstate->status[delstate->ndeltids];
664 
665  if (!BTreeTupleIsPosting(itup))
666  {
667  /* Simple case: A plain non-pivot tuple */
668  ideltid->tid = itup->t_tid;
669  ideltid->id = delstate->ndeltids;
670  istatus->idxoffnum = offnum;
671  istatus->knowndeletable = false; /* for now */
672  istatus->promising = dupinterval; /* simple rule */
673  istatus->freespace = ItemIdGetLength(itemid) + sizeof(ItemIdData);
674 
675  delstate->ndeltids++;
676  }
677  else
678  {
679  /*
680  * Complicated case: A posting list tuple.
681  *
682  * We make the conservative assumption that there can only be at
683  * most one affected logical row per posting list tuple. There
684  * will be at most one promising entry in deltids to represent
685  * this presumed lone logical row. Note that this isn't even
686  * considered unless the posting list tuple is also in an interval
687  * of duplicates -- this complicated rule is just a variant of the
688  * simple rule used to decide if plain index tuples are promising.
689  */
690  int nitem = BTreeTupleGetNPosting(itup);
691  bool firstpromising = false;
692  bool lastpromising = false;
693 
694  Assert(_bt_posting_valid(itup));
695 
696  if (dupinterval)
697  {
698  /*
699  * Complicated rule: either the first or last TID in the
700  * posting list gets marked promising (if any at all)
701  */
702  BlockNumber minblocklist,
703  midblocklist,
704  maxblocklist;
705  ItemPointer mintid,
706  midtid,
707  maxtid;
708 
709  mintid = BTreeTupleGetHeapTID(itup);
710  midtid = BTreeTupleGetPostingN(itup, nitem / 2);
711  maxtid = BTreeTupleGetMaxHeapTID(itup);
712  minblocklist = ItemPointerGetBlockNumber(mintid);
713  midblocklist = ItemPointerGetBlockNumber(midtid);
714  maxblocklist = ItemPointerGetBlockNumber(maxtid);
715 
716  /* Only entry with predominant table block can be promising */
717  firstpromising = (minblocklist == midblocklist);
718  lastpromising = (!firstpromising &&
719  midblocklist == maxblocklist);
720  }
721 
722  for (int p = 0; p < nitem; p++)
723  {
724  ItemPointer htid = BTreeTupleGetPostingN(itup, p);
725 
726  ideltid->tid = *htid;
727  ideltid->id = delstate->ndeltids;
728  istatus->idxoffnum = offnum;
729  istatus->knowndeletable = false; /* for now */
730  istatus->promising = false;
731  if ((firstpromising && p == 0) ||
732  (lastpromising && p == nitem - 1))
733  istatus->promising = true;
734  istatus->freespace = sizeof(ItemPointerData); /* at worst */
735 
736  ideltid++;
737  istatus++;
738  delstate->ndeltids++;
739  }
740  }
741  }
742 
743  if (dupinterval)
744  {
745  state->intervals[state->nintervals].nitems = state->nitems;
746  state->nintervals++;
747  }
748 
749  /* Reset state for next interval */
750  state->nhtids = 0;
751  state->nitems = 0;
752  state->phystupsize = 0;
753 }
uint32 BlockNumber
Definition: block.h:31
static Item PageGetItem(Page page, ItemId itemId)
Definition: bufpage.h:351
static ItemId PageGetItemId(Page page, OffsetNumber offsetNumber)
Definition: bufpage.h:240
int i
Definition: isn.c:73
#define ItemIdGetLength(itemId)
Definition: itemid.h:59
struct ItemIdData ItemIdData
static BlockNumber ItemPointerGetBlockNumber(const ItemPointerData *pointer)
Definition: itemptr.h:103
struct ItemPointerData ItemPointerData
IndexTupleData * IndexTuple
Definition: itup.h:53
Assert(fmt[strlen(fmt) - 1] !='\n')
static uint16 BTreeTupleGetNPosting(IndexTuple posting)
Definition: nbtree.h:518
static ItemPointer BTreeTupleGetPostingN(IndexTuple posting, int n)
Definition: nbtree.h:544
static ItemPointer BTreeTupleGetMaxHeapTID(IndexTuple itup)
Definition: nbtree.h:664
static bool BTreeTupleIsPosting(IndexTuple itup)
Definition: nbtree.h:492
static ItemPointer BTreeTupleGetHeapTID(IndexTuple itup)
Definition: nbtree.h:638
uint16 OffsetNumber
Definition: off.h:24
ItemPointerData t_tid
Definition: itup.h:37
TM_IndexStatus * status
Definition: tableam.h:247
TM_IndexDelete * deltids
Definition: tableam.h:246
ItemPointerData tid
Definition: tableam.h:205
bool knowndeletable
Definition: tableam.h:212
bool promising
Definition: tableam.h:215
int16 freespace
Definition: tableam.h:216
OffsetNumber idxoffnum
Definition: tableam.h:211
Definition: regguts.h:323

References Assert(), BTreeTupleGetHeapTID(), BTreeTupleGetMaxHeapTID(), BTreeTupleGetNPosting(), BTreeTupleGetPostingN(), BTreeTupleIsPosting(), TM_IndexDeleteOp::deltids, TM_IndexStatus::freespace, i, TM_IndexDelete::id, TM_IndexStatus::idxoffnum, ItemIdGetLength, ItemPointerGetBlockNumber(), TM_IndexStatus::knowndeletable, TM_IndexDeleteOp::ndeltids, PageGetItem(), PageGetItemId(), TM_IndexStatus::promising, TM_IndexDeleteOp::status, IndexTupleData::t_tid, and TM_IndexDelete::tid.

Referenced by _bt_bottomupdel_pass().

◆ _bt_bottomupdel_pass()

bool _bt_bottomupdel_pass ( Relation  rel,
Buffer  buf,
Relation  heapRel,
Size  newitemsz 
)

Definition at line 307 of file nbtdedup.c.

309 {
310  OffsetNumber offnum,
311  minoff,
312  maxoff;
313  Page page = BufferGetPage(buf);
314  BTPageOpaque opaque = BTPageGetOpaque(page);
316  TM_IndexDeleteOp delstate;
317  bool neverdedup;
318  int nkeyatts = IndexRelationGetNumberOfKeyAttributes(rel);
319 
320  /* Passed-in newitemsz is MAXALIGNED but does not include line pointer */
321  newitemsz += sizeof(ItemIdData);
322 
323  /* Initialize deduplication state */
325  state->deduplicate = true;
326  state->nmaxitems = 0;
327  state->maxpostingsize = BLCKSZ; /* We're not really deduplicating */
328  state->base = NULL;
329  state->baseoff = InvalidOffsetNumber;
330  state->basetupsize = 0;
331  state->htids = palloc(state->maxpostingsize);
332  state->nhtids = 0;
333  state->nitems = 0;
334  state->phystupsize = 0;
335  state->nintervals = 0;
336 
337  /*
338  * Initialize tableam state that describes bottom-up index deletion
339  * operation.
340  *
341  * We'll go on to ask the tableam to search for TIDs whose index tuples we
342  * can safely delete. The tableam will search until our leaf page space
343  * target is satisfied, or until the cost of continuing with the tableam
344  * operation seems too high. It focuses its efforts on TIDs associated
345  * with duplicate index tuples that we mark "promising".
346  *
347  * This space target is a little arbitrary. The tableam must be able to
348  * keep the costs and benefits in balance. We provide the tableam with
349  * exhaustive information about what might work, without directly
350  * concerning ourselves with avoiding work during the tableam call. Our
351  * role in costing the bottom-up deletion process is strictly advisory.
352  */
353  delstate.irel = rel;
354  delstate.iblknum = BufferGetBlockNumber(buf);
355  delstate.bottomup = true;
356  delstate.bottomupfreespace = Max(BLCKSZ / 16, newitemsz);
357  delstate.ndeltids = 0;
358  delstate.deltids = palloc(MaxTIDsPerBTreePage * sizeof(TM_IndexDelete));
359  delstate.status = palloc(MaxTIDsPerBTreePage * sizeof(TM_IndexStatus));
360 
361  minoff = P_FIRSTDATAKEY(opaque);
362  maxoff = PageGetMaxOffsetNumber(page);
363  for (offnum = minoff;
364  offnum <= maxoff;
365  offnum = OffsetNumberNext(offnum))
366  {
367  ItemId itemid = PageGetItemId(page, offnum);
368  IndexTuple itup = (IndexTuple) PageGetItem(page, itemid);
369 
370  Assert(!ItemIdIsDead(itemid));
371 
372  if (offnum == minoff)
373  {
374  /* itup starts first pending interval */
375  _bt_dedup_start_pending(state, itup, offnum);
376  }
377  else if (_bt_keep_natts_fast(rel, state->base, itup) > nkeyatts &&
378  _bt_dedup_save_htid(state, itup))
379  {
380  /* Tuple is equal; just added its TIDs to pending interval */
381  }
382  else
383  {
384  /* Finalize interval -- move its TIDs to delete state */
385  _bt_bottomupdel_finish_pending(page, state, &delstate);
386 
387  /* itup starts new pending interval */
388  _bt_dedup_start_pending(state, itup, offnum);
389  }
390  }
391  /* Finalize final interval -- move its TIDs to delete state */
392  _bt_bottomupdel_finish_pending(page, state, &delstate);
393 
394  /*
395  * We don't give up now in the event of having few (or even zero)
396  * promising tuples for the tableam because it's not up to us as the index
397  * AM to manage costs (note that the tableam might have heuristics of its
398  * own that work out what to do). We should at least avoid having our
399  * caller do a useless deduplication pass after we return in the event of
400  * zero promising tuples, though.
401  */
402  neverdedup = false;
403  if (state->nintervals == 0)
404  neverdedup = true;
405 
406  pfree(state->htids);
407  pfree(state);
408 
409  /* Ask tableam which TIDs are deletable, then physically delete them */
410  _bt_delitems_delete_check(rel, buf, heapRel, &delstate);
411 
412  pfree(delstate.deltids);
413  pfree(delstate.status);
414 
415  /* Report "success" to caller unconditionally to avoid deduplication */
416  if (neverdedup)
417  return true;
418 
419  /* Don't dedup when we won't end up back here any time soon anyway */
420  return PageGetExactFreeSpace(page) >= Max(BLCKSZ / 24, newitemsz);
421 }
BlockNumber BufferGetBlockNumber(Buffer buffer)
Definition: bufmgr.c:3377
static Page BufferGetPage(Buffer buffer)
Definition: bufmgr.h:350
Size PageGetExactFreeSpace(Page page)
Definition: bufpage.c:958
Pointer Page
Definition: bufpage.h:78
static OffsetNumber PageGetMaxOffsetNumber(Page page)
Definition: bufpage.h:369
#define Max(x, y)
Definition: c.h:985
#define ItemIdIsDead(itemId)
Definition: itemid.h:113
void pfree(void *pointer)
Definition: mcxt.c:1508
void * palloc(Size size)
Definition: mcxt.c:1304
bool _bt_dedup_save_htid(BTDedupState state, IndexTuple itup)
Definition: nbtdedup.c:484
void _bt_dedup_start_pending(BTDedupState state, IndexTuple base, OffsetNumber baseoff)
Definition: nbtdedup.c:433
static void _bt_bottomupdel_finish_pending(Page page, BTDedupState state, TM_IndexDeleteOp *delstate)
Definition: nbtdedup.c:648
void _bt_delitems_delete_check(Relation rel, Buffer buf, Relation heapRel, TM_IndexDeleteOp *delstate)
Definition: nbtpage.c:1513
#define BTPageGetOpaque(page)
Definition: nbtree.h:73
#define MaxTIDsPerBTreePage
Definition: nbtree.h:185
#define P_FIRSTDATAKEY(opaque)
Definition: nbtree.h:369
BTDedupStateData * BTDedupState
Definition: nbtree.h:893
int _bt_keep_natts_fast(Relation rel, IndexTuple lastleft, IndexTuple firstright)
Definition: nbtutils.c:2475
#define InvalidOffsetNumber
Definition: off.h:26
#define OffsetNumberNext(offsetNumber)
Definition: off.h:52
static char * buf
Definition: pg_test_fsync.c:73
#define IndexRelationGetNumberOfKeyAttributes(relation)
Definition: rel.h:526
int bottomupfreespace
Definition: tableam.h:242
Relation irel
Definition: tableam.h:239
BlockNumber iblknum
Definition: tableam.h:240

References _bt_bottomupdel_finish_pending(), _bt_dedup_save_htid(), _bt_dedup_start_pending(), _bt_delitems_delete_check(), _bt_keep_natts_fast(), Assert(), TM_IndexDeleteOp::bottomup, TM_IndexDeleteOp::bottomupfreespace, BTPageGetOpaque, buf, BufferGetBlockNumber(), BufferGetPage(), TM_IndexDeleteOp::deltids, TM_IndexDeleteOp::iblknum, IndexRelationGetNumberOfKeyAttributes, InvalidOffsetNumber, TM_IndexDeleteOp::irel, ItemIdIsDead, Max, MaxTIDsPerBTreePage, TM_IndexDeleteOp::ndeltids, OffsetNumberNext, P_FIRSTDATAKEY, PageGetExactFreeSpace(), PageGetItem(), PageGetItemId(), PageGetMaxOffsetNumber(), palloc(), pfree(), and TM_IndexDeleteOp::status.

Referenced by _bt_delete_or_dedup_one_page().

◆ _bt_dedup_finish_pending()

Size _bt_dedup_finish_pending ( Page  newpage,
BTDedupState  state 
)

Definition at line 555 of file nbtdedup.c.

556 {
557  OffsetNumber tupoff;
558  Size tuplesz;
559  Size spacesaving;
560 
561  Assert(state->nitems > 0);
562  Assert(state->nitems <= state->nhtids);
563  Assert(state->intervals[state->nintervals].baseoff == state->baseoff);
564 
565  tupoff = OffsetNumberNext(PageGetMaxOffsetNumber(newpage));
566  if (state->nitems == 1)
567  {
568  /* Use original, unchanged base tuple */
569  tuplesz = IndexTupleSize(state->base);
570  Assert(tuplesz == MAXALIGN(IndexTupleSize(state->base)));
571  Assert(tuplesz <= BTMaxItemSize(newpage));
572  if (PageAddItem(newpage, (Item) state->base, tuplesz, tupoff,
573  false, false) == InvalidOffsetNumber)
574  elog(ERROR, "deduplication failed to add tuple to page");
575 
576  spacesaving = 0;
577  }
578  else
579  {
580  IndexTuple final;
581 
582  /* Form a tuple with a posting list */
583  final = _bt_form_posting(state->base, state->htids, state->nhtids);
584  tuplesz = IndexTupleSize(final);
585  Assert(tuplesz <= state->maxpostingsize);
586 
587  /* Save final number of items for posting list */
588  state->intervals[state->nintervals].nitems = state->nitems;
589 
590  Assert(tuplesz == MAXALIGN(IndexTupleSize(final)));
591  Assert(tuplesz <= BTMaxItemSize(newpage));
592  if (PageAddItem(newpage, (Item) final, tuplesz, tupoff, false,
593  false) == InvalidOffsetNumber)
594  elog(ERROR, "deduplication failed to add tuple to page");
595 
596  pfree(final);
597  spacesaving = state->phystupsize - (tuplesz + sizeof(ItemIdData));
598  /* Increment nintervals, since we wrote a new posting list tuple */
599  state->nintervals++;
600  Assert(spacesaving > 0 && spacesaving < BLCKSZ);
601  }
602 
603  /* Reset state for next pending posting list */
604  state->nhtids = 0;
605  state->nitems = 0;
606  state->phystupsize = 0;
607 
608  return spacesaving;
609 }
#define PageAddItem(page, item, size, offsetNumber, overwrite, is_heap)
Definition: bufpage.h:468
#define MAXALIGN(LEN)
Definition: c.h:798
size_t Size
Definition: c.h:592
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:224
Pointer Item
Definition: item.h:17
#define IndexTupleSize(itup)
Definition: itup.h:70
IndexTuple _bt_form_posting(IndexTuple base, ItemPointer htids, int nhtids)
Definition: nbtdedup.c:864
#define BTMaxItemSize(page)
Definition: nbtree.h:164

References _bt_form_posting(), Assert(), BTMaxItemSize, elog, ERROR, IndexTupleSize, InvalidOffsetNumber, MAXALIGN, OffsetNumberNext, PageAddItem, PageGetMaxOffsetNumber(), and pfree().

Referenced by _bt_dedup_pass(), and btree_xlog_dedup().

◆ _bt_dedup_pass()

void _bt_dedup_pass ( Relation  rel,
Buffer  buf,
IndexTuple  newitem,
Size  newitemsz,
bool  bottomupdedup 
)

Definition at line 58 of file nbtdedup.c.

60 {
61  OffsetNumber offnum,
62  minoff,
63  maxoff;
64  Page page = BufferGetPage(buf);
65  BTPageOpaque opaque = BTPageGetOpaque(page);
66  Page newpage;
68  Size pagesaving PG_USED_FOR_ASSERTS_ONLY = 0;
69  bool singlevalstrat = false;
70  int nkeyatts = IndexRelationGetNumberOfKeyAttributes(rel);
71 
72  /* Passed-in newitemsz is MAXALIGNED but does not include line pointer */
73  newitemsz += sizeof(ItemIdData);
74 
75  /*
76  * Initialize deduplication state.
77  *
78  * It would be possible for maxpostingsize (limit on posting list tuple
79  * size) to be set to one third of the page. However, it seems like a
80  * good idea to limit the size of posting lists to one sixth of a page.
81  * That ought to leave us with a good split point when pages full of
82  * duplicates can be split several times.
83  */
85  state->deduplicate = true;
86  state->nmaxitems = 0;
87  state->maxpostingsize = Min(BTMaxItemSize(page) / 2, INDEX_SIZE_MASK);
88  /* Metadata about base tuple of current pending posting list */
89  state->base = NULL;
90  state->baseoff = InvalidOffsetNumber;
91  state->basetupsize = 0;
92  /* Metadata about current pending posting list TIDs */
93  state->htids = palloc(state->maxpostingsize);
94  state->nhtids = 0;
95  state->nitems = 0;
96  /* Size of all physical tuples to be replaced by pending posting list */
97  state->phystupsize = 0;
98  /* nintervals should be initialized to zero */
99  state->nintervals = 0;
100 
101  minoff = P_FIRSTDATAKEY(opaque);
102  maxoff = PageGetMaxOffsetNumber(page);
103 
104  /*
105  * Consider applying "single value" strategy, though only if the page
106  * seems likely to be split in the near future
107  */
108  if (!bottomupdedup)
109  singlevalstrat = _bt_do_singleval(rel, page, state, minoff, newitem);
110 
111  /*
112  * Deduplicate items from page, and write them to newpage.
113  *
114  * Copy the original page's LSN into newpage copy. This will become the
115  * updated version of the page. We need this because XLogInsert will
116  * examine the LSN and possibly dump it in a page image.
117  */
118  newpage = PageGetTempPageCopySpecial(page);
119  PageSetLSN(newpage, PageGetLSN(page));
120 
121  /* Copy high key, if any */
122  if (!P_RIGHTMOST(opaque))
123  {
124  ItemId hitemid = PageGetItemId(page, P_HIKEY);
125  Size hitemsz = ItemIdGetLength(hitemid);
126  IndexTuple hitem = (IndexTuple) PageGetItem(page, hitemid);
127 
128  if (PageAddItem(newpage, (Item) hitem, hitemsz, P_HIKEY,
129  false, false) == InvalidOffsetNumber)
130  elog(ERROR, "deduplication failed to add highkey");
131  }
132 
133  for (offnum = minoff;
134  offnum <= maxoff;
135  offnum = OffsetNumberNext(offnum))
136  {
137  ItemId itemid = PageGetItemId(page, offnum);
138  IndexTuple itup = (IndexTuple) PageGetItem(page, itemid);
139 
140  Assert(!ItemIdIsDead(itemid));
141 
142  if (offnum == minoff)
143  {
144  /*
145  * No previous/base tuple for the data item -- use the data item
146  * as base tuple of pending posting list
147  */
148  _bt_dedup_start_pending(state, itup, offnum);
149  }
150  else if (state->deduplicate &&
151  _bt_keep_natts_fast(rel, state->base, itup) > nkeyatts &&
152  _bt_dedup_save_htid(state, itup))
153  {
154  /*
155  * Tuple is equal to base tuple of pending posting list. Heap
156  * TID(s) for itup have been saved in state.
157  */
158  }
159  else
160  {
161  /*
162  * Tuple is not equal to pending posting list tuple, or
163  * _bt_dedup_save_htid() opted to not merge current item into
164  * pending posting list for some other reason (e.g., adding more
165  * TIDs would have caused posting list to exceed current
166  * maxpostingsize).
167  *
168  * If state contains pending posting list with more than one item,
169  * form new posting tuple and add it to our temp page (newpage).
170  * Else add pending interval's base tuple to the temp page as-is.
171  */
172  pagesaving += _bt_dedup_finish_pending(newpage, state);
173 
174  if (singlevalstrat)
175  {
176  /*
177  * Single value strategy's extra steps.
178  *
179  * Lower maxpostingsize for sixth and final large posting list
180  * tuple at the point where 5 maxpostingsize-capped tuples
181  * have either been formed or observed.
182  *
183  * When a sixth maxpostingsize-capped item is formed/observed,
184  * stop merging together tuples altogether. The few tuples
185  * that remain at the end of the page won't be merged together
186  * at all (at least not until after a future page split takes
187  * place, when this page's newly allocated right sibling page
188  * gets its first deduplication pass).
189  */
190  if (state->nmaxitems == 5)
191  _bt_singleval_fillfactor(page, state, newitemsz);
192  else if (state->nmaxitems == 6)
193  {
194  state->deduplicate = false;
195  singlevalstrat = false; /* won't be back here */
196  }
197  }
198 
199  /* itup starts new pending posting list */
200  _bt_dedup_start_pending(state, itup, offnum);
201  }
202  }
203 
204  /* Handle the last item */
205  pagesaving += _bt_dedup_finish_pending(newpage, state);
206 
207  /*
208  * If no items suitable for deduplication were found, newpage must be
209  * exactly the same as the original page, so just return from function.
210  *
211  * We could determine whether or not to proceed on the basis the space
212  * savings being sufficient to avoid an immediate page split instead. We
213  * don't do that because there is some small value in nbtsplitloc.c always
214  * operating against a page that is fully deduplicated (apart from
215  * newitem). Besides, most of the cost has already been paid.
216  */
217  if (state->nintervals == 0)
218  {
219  /* cannot leak memory here */
220  pfree(newpage);
221  pfree(state->htids);
222  pfree(state);
223  return;
224  }
225 
226  /*
227  * By here, it's clear that deduplication will definitely go ahead.
228  *
229  * Clear the BTP_HAS_GARBAGE page flag. The index must be a heapkeyspace
230  * index, and as such we'll never pay attention to BTP_HAS_GARBAGE anyway.
231  * But keep things tidy.
232  */
233  if (P_HAS_GARBAGE(opaque))
234  {
235  BTPageOpaque nopaque = BTPageGetOpaque(newpage);
236 
237  nopaque->btpo_flags &= ~BTP_HAS_GARBAGE;
238  }
239 
241 
242  PageRestoreTempPage(newpage, page);
244 
245  /* XLOG stuff */
246  if (RelationNeedsWAL(rel))
247  {
248  XLogRecPtr recptr;
249  xl_btree_dedup xlrec_dedup;
250 
251  xlrec_dedup.nintervals = state->nintervals;
252 
253  XLogBeginInsert();
255  XLogRegisterData((char *) &xlrec_dedup, SizeOfBtreeDedup);
256 
257  /*
258  * The intervals array is not in the buffer, but pretend that it is.
259  * When XLogInsert stores the whole buffer, the array need not be
260  * stored too.
261  */
262  XLogRegisterBufData(0, (char *) state->intervals,
263  state->nintervals * sizeof(BTDedupInterval));
264 
265  recptr = XLogInsert(RM_BTREE_ID, XLOG_BTREE_DEDUP);
266 
267  PageSetLSN(page, recptr);
268  }
269 
271 
272  /* Local space accounting should agree with page accounting */
273  Assert(pagesaving < newitemsz || PageGetExactFreeSpace(page) >= newitemsz);
274 
275  /* cannot leak memory here */
276  pfree(state->htids);
277  pfree(state);
278 }
void MarkBufferDirty(Buffer buffer)
Definition: bufmgr.c:2189
void PageRestoreTempPage(Page tempPage, Page oldPage)
Definition: bufpage.c:424
Page PageGetTempPageCopySpecial(Page page)
Definition: bufpage.c:402
static void PageSetLSN(Page page, XLogRecPtr lsn)
Definition: bufpage.h:388
static XLogRecPtr PageGetLSN(Page page)
Definition: bufpage.h:383
#define Min(x, y)
Definition: c.h:991
#define PG_USED_FOR_ASSERTS_ONLY
Definition: c.h:169
#define INDEX_SIZE_MASK
Definition: itup.h:65
#define START_CRIT_SECTION()
Definition: miscadmin.h:149
#define END_CRIT_SECTION()
Definition: miscadmin.h:151
static bool _bt_do_singleval(Relation rel, Page page, BTDedupState state, OffsetNumber minoff, IndexTuple newitem)
Definition: nbtdedup.c:782
Size _bt_dedup_finish_pending(Page newpage, BTDedupState state)
Definition: nbtdedup.c:555
static void _bt_singleval_fillfactor(Page page, BTDedupState state, Size newitemsz)
Definition: nbtdedup.c:822
#define P_HIKEY
Definition: nbtree.h:367
#define P_HAS_GARBAGE(opaque)
Definition: nbtree.h:226
#define BTP_HAS_GARBAGE
Definition: nbtree.h:82
#define P_RIGHTMOST(opaque)
Definition: nbtree.h:219
#define XLOG_BTREE_DEDUP
Definition: nbtxlog.h:33
#define SizeOfBtreeDedup
Definition: nbtxlog.h:174
#define RelationNeedsWAL(relation)
Definition: rel.h:630
uint16 btpo_flags
Definition: nbtree.h:67
uint16 nintervals
Definition: nbtxlog.h:169
uint64 XLogRecPtr
Definition: xlogdefs.h:21
void XLogRegisterData(char *data, uint32 len)
Definition: xloginsert.c:364
XLogRecPtr XLogInsert(RmgrId rmid, uint8 info)
Definition: xloginsert.c:474
void XLogRegisterBufData(uint8 block_id, char *data, uint32 len)
Definition: xloginsert.c:405
void XLogRegisterBuffer(uint8 block_id, Buffer buffer, uint8 flags)
Definition: xloginsert.c:242
void XLogBeginInsert(void)
Definition: xloginsert.c:149
#define REGBUF_STANDARD
Definition: xloginsert.h:34

References _bt_dedup_finish_pending(), _bt_dedup_save_htid(), _bt_dedup_start_pending(), _bt_do_singleval(), _bt_keep_natts_fast(), _bt_singleval_fillfactor(), Assert(), BTMaxItemSize, BTP_HAS_GARBAGE, BTPageGetOpaque, BTPageOpaqueData::btpo_flags, buf, BufferGetPage(), elog, END_CRIT_SECTION, ERROR, INDEX_SIZE_MASK, IndexRelationGetNumberOfKeyAttributes, InvalidOffsetNumber, ItemIdGetLength, ItemIdIsDead, MarkBufferDirty(), Min, xl_btree_dedup::nintervals, OffsetNumberNext, P_FIRSTDATAKEY, P_HAS_GARBAGE, P_HIKEY, P_RIGHTMOST, PageAddItem, PageGetExactFreeSpace(), PageGetItem(), PageGetItemId(), PageGetLSN(), PageGetMaxOffsetNumber(), PageGetTempPageCopySpecial(), PageRestoreTempPage(), PageSetLSN(), palloc(), pfree(), PG_USED_FOR_ASSERTS_ONLY, REGBUF_STANDARD, RelationNeedsWAL, SizeOfBtreeDedup, START_CRIT_SECTION, XLOG_BTREE_DEDUP, XLogBeginInsert(), XLogInsert(), XLogRegisterBufData(), XLogRegisterBuffer(), and XLogRegisterData().

Referenced by _bt_delete_or_dedup_one_page().

◆ _bt_dedup_save_htid()

bool _bt_dedup_save_htid ( BTDedupState  state,
IndexTuple  itup 
)

Definition at line 484 of file nbtdedup.c.

485 {
486  int nhtids;
487  ItemPointer htids;
488  Size mergedtupsz;
489 
490  Assert(!BTreeTupleIsPivot(itup));
491 
492  if (!BTreeTupleIsPosting(itup))
493  {
494  nhtids = 1;
495  htids = &itup->t_tid;
496  }
497  else
498  {
499  nhtids = BTreeTupleGetNPosting(itup);
500  htids = BTreeTupleGetPosting(itup);
501  }
502 
503  /*
504  * Don't append (have caller finish pending posting list as-is) if
505  * appending heap TID(s) from itup would put us over maxpostingsize limit.
506  *
507  * This calculation needs to match the code used within _bt_form_posting()
508  * for new posting list tuples.
509  */
510  mergedtupsz = MAXALIGN(state->basetupsize +
511  (state->nhtids + nhtids) * sizeof(ItemPointerData));
512 
513  if (mergedtupsz > state->maxpostingsize)
514  {
515  /*
516  * Count this as an oversized item for single value strategy, though
517  * only when there are 50 TIDs in the final posting list tuple. This
518  * limit (which is fairly arbitrary) avoids confusion about how many
519  * 1/6 of a page tuples have been encountered/created by the current
520  * deduplication pass.
521  *
522  * Note: We deliberately don't consider which deduplication pass
523  * merged together tuples to create this item (could be a previous
524  * deduplication pass, or current pass). See _bt_do_singleval()
525  * comments.
526  */
527  if (state->nhtids > 50)
528  state->nmaxitems++;
529 
530  return false;
531  }
532 
533  /*
534  * Save heap TIDs to pending posting list tuple -- itup can be merged into
535  * pending posting list
536  */
537  state->nitems++;
538  memcpy(state->htids + state->nhtids, htids,
539  sizeof(ItemPointerData) * nhtids);
540  state->nhtids += nhtids;
541  state->phystupsize += MAXALIGN(IndexTupleSize(itup)) + sizeof(ItemIdData);
542 
543  return true;
544 }
static bool BTreeTupleIsPivot(IndexTuple itup)
Definition: nbtree.h:480
static ItemPointer BTreeTupleGetPosting(IndexTuple posting)
Definition: nbtree.h:537

References Assert(), BTreeTupleGetNPosting(), BTreeTupleGetPosting(), BTreeTupleIsPivot(), BTreeTupleIsPosting(), IndexTupleSize, MAXALIGN, and IndexTupleData::t_tid.

Referenced by _bt_bottomupdel_pass(), _bt_dedup_pass(), _bt_load(), and btree_xlog_dedup().

◆ _bt_dedup_start_pending()

void _bt_dedup_start_pending ( BTDedupState  state,
IndexTuple  base,
OffsetNumber  baseoff 
)

Definition at line 433 of file nbtdedup.c.

435 {
436  Assert(state->nhtids == 0);
437  Assert(state->nitems == 0);
438  Assert(!BTreeTupleIsPivot(base));
439 
440  /*
441  * Copy heap TID(s) from new base tuple for new candidate posting list
442  * into working state's array
443  */
444  if (!BTreeTupleIsPosting(base))
445  {
446  memcpy(state->htids, &base->t_tid, sizeof(ItemPointerData));
447  state->nhtids = 1;
448  state->basetupsize = IndexTupleSize(base);
449  }
450  else
451  {
452  int nposting;
453 
454  nposting = BTreeTupleGetNPosting(base);
455  memcpy(state->htids, BTreeTupleGetPosting(base),
456  sizeof(ItemPointerData) * nposting);
457  state->nhtids = nposting;
458  /* basetupsize should not include existing posting list */
459  state->basetupsize = BTreeTupleGetPostingOffset(base);
460  }
461 
462  /*
463  * Save new base tuple itself -- it'll be needed if we actually create a
464  * new posting list from new pending posting list.
465  *
466  * Must maintain physical size of all existing tuples (including line
467  * pointer overhead) so that we can calculate space savings on page.
468  */
469  state->nitems = 1;
470  state->base = base;
471  state->baseoff = baseoff;
472  state->phystupsize = MAXALIGN(IndexTupleSize(base)) + sizeof(ItemIdData);
473  /* Also save baseoff in pending state for interval */
474  state->intervals[state->nintervals].baseoff = state->baseoff;
475 }
static uint32 BTreeTupleGetPostingOffset(IndexTuple posting)
Definition: nbtree.h:529

References Assert(), BTreeTupleGetNPosting(), BTreeTupleGetPosting(), BTreeTupleGetPostingOffset(), BTreeTupleIsPivot(), BTreeTupleIsPosting(), IndexTupleSize, MAXALIGN, and IndexTupleData::t_tid.

Referenced by _bt_bottomupdel_pass(), _bt_dedup_pass(), _bt_load(), and btree_xlog_dedup().

◆ _bt_do_singleval()

static bool _bt_do_singleval ( Relation  rel,
Page  page,
BTDedupState  state,
OffsetNumber  minoff,
IndexTuple  newitem 
)
static

Definition at line 782 of file nbtdedup.c.

784 {
785  int nkeyatts = IndexRelationGetNumberOfKeyAttributes(rel);
786  ItemId itemid;
787  IndexTuple itup;
788 
789  itemid = PageGetItemId(page, minoff);
790  itup = (IndexTuple) PageGetItem(page, itemid);
791 
792  if (_bt_keep_natts_fast(rel, newitem, itup) > nkeyatts)
793  {
794  itemid = PageGetItemId(page, PageGetMaxOffsetNumber(page));
795  itup = (IndexTuple) PageGetItem(page, itemid);
796 
797  if (_bt_keep_natts_fast(rel, newitem, itup) > nkeyatts)
798  return true;
799  }
800 
801  return false;
802 }

References _bt_keep_natts_fast(), IndexRelationGetNumberOfKeyAttributes, PageGetItem(), PageGetItemId(), and PageGetMaxOffsetNumber().

Referenced by _bt_dedup_pass().

◆ _bt_form_posting()

IndexTuple _bt_form_posting ( IndexTuple  base,
ItemPointer  htids,
int  nhtids 
)

Definition at line 864 of file nbtdedup.c.

865 {
866  uint32 keysize,
867  newsize;
868  IndexTuple itup;
869 
870  if (BTreeTupleIsPosting(base))
871  keysize = BTreeTupleGetPostingOffset(base);
872  else
873  keysize = IndexTupleSize(base);
874 
875  Assert(!BTreeTupleIsPivot(base));
876  Assert(nhtids > 0 && nhtids <= PG_UINT16_MAX);
877  Assert(keysize == MAXALIGN(keysize));
878 
879  /* Determine final size of new tuple */
880  if (nhtids > 1)
881  newsize = MAXALIGN(keysize +
882  nhtids * sizeof(ItemPointerData));
883  else
884  newsize = keysize;
885 
886  Assert(newsize <= INDEX_SIZE_MASK);
887  Assert(newsize == MAXALIGN(newsize));
888 
889  /* Allocate memory using palloc0() (matches index_form_tuple()) */
890  itup = palloc0(newsize);
891  memcpy(itup, base, keysize);
892  itup->t_info &= ~INDEX_SIZE_MASK;
893  itup->t_info |= newsize;
894  if (nhtids > 1)
895  {
896  /* Form posting list tuple */
897  BTreeTupleSetPosting(itup, nhtids, keysize);
898  memcpy(BTreeTupleGetPosting(itup), htids,
899  sizeof(ItemPointerData) * nhtids);
900  Assert(_bt_posting_valid(itup));
901  }
902  else
903  {
904  /* Form standard non-pivot tuple */
905  itup->t_info &= ~INDEX_ALT_TID_MASK;
906  ItemPointerCopy(htids, &itup->t_tid);
908  }
909 
910  return itup;
911 }
unsigned int uint32
Definition: c.h:493
#define PG_UINT16_MAX
Definition: c.h:574
static void ItemPointerCopy(const ItemPointerData *fromPointer, ItemPointerData *toPointer)
Definition: itemptr.h:172
static bool ItemPointerIsValid(const ItemPointerData *pointer)
Definition: itemptr.h:83
void * palloc0(Size size)
Definition: mcxt.c:1334
static void BTreeTupleSetPosting(IndexTuple itup, uint16 nhtids, int postingoffset)
Definition: nbtree.h:504
#define INDEX_ALT_TID_MASK
Definition: nbtree.h:459
unsigned short t_info
Definition: itup.h:49

References Assert(), BTreeTupleGetPosting(), BTreeTupleGetPostingOffset(), BTreeTupleIsPivot(), BTreeTupleIsPosting(), BTreeTupleSetPosting(), INDEX_ALT_TID_MASK, INDEX_SIZE_MASK, IndexTupleSize, ItemPointerCopy(), ItemPointerIsValid(), MAXALIGN, palloc0(), PG_UINT16_MAX, IndexTupleData::t_info, and IndexTupleData::t_tid.

Referenced by _bt_dedup_finish_pending(), _bt_sort_dedup_finish_pending(), and bt_posting_plain_tuple().

◆ _bt_singleval_fillfactor()

static void _bt_singleval_fillfactor ( Page  page,
BTDedupState  state,
Size  newitemsz 
)
static

Definition at line 822 of file nbtdedup.c.

823 {
824  Size leftfree;
825  int reduction;
826 
827  /* This calculation needs to match nbtsplitloc.c */
828  leftfree = PageGetPageSize(page) - SizeOfPageHeaderData -
829  MAXALIGN(sizeof(BTPageOpaqueData));
830  /* Subtract size of new high key (includes pivot heap TID space) */
831  leftfree -= newitemsz + MAXALIGN(sizeof(ItemPointerData));
832 
833  /*
834  * Reduce maxpostingsize by an amount equal to target free space on left
835  * half of page
836  */
837  reduction = leftfree * ((100 - BTREE_SINGLEVAL_FILLFACTOR) / 100.0);
838  if (state->maxpostingsize > reduction)
839  state->maxpostingsize -= reduction;
840  else
841  state->maxpostingsize = 0;
842 }
static Size PageGetPageSize(Page page)
Definition: bufpage.h:273
#define SizeOfPageHeaderData
Definition: bufpage.h:213
#define BTREE_SINGLEVAL_FILLFACTOR
Definition: nbtree.h:202

References BTREE_SINGLEVAL_FILLFACTOR, MAXALIGN, PageGetPageSize(), and SizeOfPageHeaderData.

Referenced by _bt_dedup_pass().

◆ _bt_swap_posting()

IndexTuple _bt_swap_posting ( IndexTuple  newitem,
IndexTuple  oposting,
int  postingoff 
)

Definition at line 1022 of file nbtdedup.c.

1023 {
1024  int nhtids;
1025  char *replacepos;
1026  char *replaceposright;
1027  Size nmovebytes;
1028  IndexTuple nposting;
1029 
1030  nhtids = BTreeTupleGetNPosting(oposting);
1031  Assert(_bt_posting_valid(oposting));
1032 
1033  /*
1034  * The postingoff argument originated as a _bt_binsrch_posting() return
1035  * value. It will be 0 in the event of corruption that makes a leaf page
1036  * contain a non-pivot tuple that's somehow identical to newitem (no two
1037  * non-pivot tuples should ever have the same TID). This has been known
1038  * to happen in the field from time to time.
1039  *
1040  * Perform a basic sanity check to catch this case now.
1041  */
1042  if (!(postingoff > 0 && postingoff < nhtids))
1043  elog(ERROR, "posting list tuple with %d items cannot be split at offset %d",
1044  nhtids, postingoff);
1045 
1046  /*
1047  * Move item pointers in posting list to make a gap for the new item's
1048  * heap TID. We shift TIDs one place to the right, losing original
1049  * rightmost TID. (nmovebytes must not include TIDs to the left of
1050  * postingoff, nor the existing rightmost/max TID that gets overwritten.)
1051  */
1052  nposting = CopyIndexTuple(oposting);
1053  replacepos = (char *) BTreeTupleGetPostingN(nposting, postingoff);
1054  replaceposright = (char *) BTreeTupleGetPostingN(nposting, postingoff + 1);
1055  nmovebytes = (nhtids - postingoff - 1) * sizeof(ItemPointerData);
1056  memmove(replaceposright, replacepos, nmovebytes);
1057 
1058  /* Fill the gap at postingoff with TID of new item (original new TID) */
1059  Assert(!BTreeTupleIsPivot(newitem) && !BTreeTupleIsPosting(newitem));
1060  ItemPointerCopy(&newitem->t_tid, (ItemPointer) replacepos);
1061 
1062  /* Now copy oposting's rightmost/max TID into new item (final new TID) */
1063  ItemPointerCopy(BTreeTupleGetMaxHeapTID(oposting), &newitem->t_tid);
1064 
1066  BTreeTupleGetHeapTID(newitem)) < 0);
1067  Assert(_bt_posting_valid(nposting));
1068 
1069  return nposting;
1070 }
IndexTuple CopyIndexTuple(IndexTuple source)
Definition: indextuple.c:547
int32 ItemPointerCompare(ItemPointer arg1, ItemPointer arg2)
Definition: itemptr.c:51

References Assert(), BTreeTupleGetHeapTID(), BTreeTupleGetMaxHeapTID(), BTreeTupleGetNPosting(), BTreeTupleGetPostingN(), BTreeTupleIsPivot(), BTreeTupleIsPosting(), CopyIndexTuple(), elog, ERROR, ItemPointerCompare(), ItemPointerCopy(), and IndexTupleData::t_tid.

Referenced by _bt_insertonpg(), btree_xlog_insert(), and btree_xlog_split().

◆ _bt_update_posting()

void _bt_update_posting ( BTVacuumPosting  vacposting)

Definition at line 924 of file nbtdedup.c.

925 {
926  IndexTuple origtuple = vacposting->itup;
927  uint32 keysize,
928  newsize;
929  IndexTuple itup;
930  int nhtids;
931  int ui,
932  d;
933  ItemPointer htids;
934 
935  nhtids = BTreeTupleGetNPosting(origtuple) - vacposting->ndeletedtids;
936 
937  Assert(_bt_posting_valid(origtuple));
938  Assert(nhtids > 0 && nhtids < BTreeTupleGetNPosting(origtuple));
939 
940  /*
941  * Determine final size of new tuple.
942  *
943  * This calculation needs to match the code used within _bt_form_posting()
944  * for new posting list tuples. We avoid calling _bt_form_posting() here
945  * to save ourselves a second memory allocation for a htids workspace.
946  */
947  keysize = BTreeTupleGetPostingOffset(origtuple);
948  if (nhtids > 1)
949  newsize = MAXALIGN(keysize +
950  nhtids * sizeof(ItemPointerData));
951  else
952  newsize = keysize;
953 
954  Assert(newsize <= INDEX_SIZE_MASK);
955  Assert(newsize == MAXALIGN(newsize));
956 
957  /* Allocate memory using palloc0() (matches index_form_tuple()) */
958  itup = palloc0(newsize);
959  memcpy(itup, origtuple, keysize);
960  itup->t_info &= ~INDEX_SIZE_MASK;
961  itup->t_info |= newsize;
962 
963  if (nhtids > 1)
964  {
965  /* Form posting list tuple */
966  BTreeTupleSetPosting(itup, nhtids, keysize);
967  htids = BTreeTupleGetPosting(itup);
968  }
969  else
970  {
971  /* Form standard non-pivot tuple */
972  itup->t_info &= ~INDEX_ALT_TID_MASK;
973  htids = &itup->t_tid;
974  }
975 
976  ui = 0;
977  d = 0;
978  for (int i = 0; i < BTreeTupleGetNPosting(origtuple); i++)
979  {
980  if (d < vacposting->ndeletedtids && vacposting->deletetids[d] == i)
981  {
982  d++;
983  continue;
984  }
985  htids[ui++] = *BTreeTupleGetPostingN(origtuple, i);
986  }
987  Assert(ui == nhtids);
988  Assert(d == vacposting->ndeletedtids);
989  Assert(nhtids == 1 || _bt_posting_valid(itup));
990  Assert(nhtids > 1 || ItemPointerIsValid(&itup->t_tid));
991 
992  /* vacposting arg's itup will now point to updated version */
993  vacposting->itup = itup;
994 }
uint16 deletetids[FLEXIBLE_ARRAY_MEMBER]
Definition: nbtree.h:911
uint16 ndeletedtids
Definition: nbtree.h:910
IndexTuple itup
Definition: nbtree.h:906

References Assert(), BTreeTupleGetNPosting(), BTreeTupleGetPosting(), BTreeTupleGetPostingN(), BTreeTupleGetPostingOffset(), BTreeTupleSetPosting(), BTVacuumPostingData::deletetids, i, INDEX_ALT_TID_MASK, INDEX_SIZE_MASK, ItemPointerIsValid(), BTVacuumPostingData::itup, MAXALIGN, BTVacuumPostingData::ndeletedtids, palloc0(), IndexTupleData::t_info, and IndexTupleData::t_tid.

Referenced by _bt_delitems_update(), and btree_xlog_updates().