PostgreSQL Source Code  git master
blinsert.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * blinsert.c
4  * Bloom index build and insert functions.
5  *
6  * Copyright (c) 2016-2018, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * contrib/bloom/blinsert.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 #include "postgres.h"
14 
15 #include "access/genam.h"
16 #include "access/generic_xlog.h"
17 #include "catalog/index.h"
18 #include "miscadmin.h"
19 #include "storage/bufmgr.h"
20 #include "storage/indexfsm.h"
21 #include "storage/smgr.h"
22 #include "utils/memutils.h"
23 #include "utils/rel.h"
24 
25 #include "bloom.h"
26 
28 
29 /*
30  * State of bloom index build. We accumulate one page data here before
31  * flushing it to buffer manager.
32  */
33 typedef struct
34 {
35  BloomState blstate; /* bloom index state */
36  int64 indtuples; /* total number of tuples indexed */
37  MemoryContext tmpCtx; /* temporary memory context reset after each
38  * tuple */
39  char data[BLCKSZ]; /* cached page */
40  int count; /* number of tuples in cached page */
42 
43 /*
44  * Flush page cached in BloomBuildState.
45  */
46 static void
48 {
49  Page page;
50  Buffer buffer = BloomNewBuffer(index);
52 
53  state = GenericXLogStart(index);
55  memcpy(page, buildstate->data, BLCKSZ);
56  GenericXLogFinish(state);
57  UnlockReleaseBuffer(buffer);
58 }
59 
60 /*
61  * (Re)initialize cached page in BloomBuildState.
62  */
63 static void
65 {
66  memset(buildstate->data, 0, BLCKSZ);
67  BloomInitPage(buildstate->data, 0);
68  buildstate->count = 0;
69 }
70 
71 /*
72  * Per-tuple callback from IndexBuildHeapScan.
73  */
74 static void
76  bool *isnull, bool tupleIsAlive, void *state)
77 {
78  BloomBuildState *buildstate = (BloomBuildState *) state;
79  MemoryContext oldCtx;
80  BloomTuple *itup;
81 
82  oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
83 
84  itup = BloomFormTuple(&buildstate->blstate, &htup->t_self, values, isnull);
85 
86  /* Try to add next item to cached page */
87  if (BloomPageAddItem(&buildstate->blstate, buildstate->data, itup))
88  {
89  /* Next item was added successfully */
90  buildstate->count++;
91  }
92  else
93  {
94  /* Cached page is full, flush it out and make a new one */
95  flushCachedPage(index, buildstate);
96 
98 
99  initCachedPage(buildstate);
100 
101  if (!BloomPageAddItem(&buildstate->blstate, buildstate->data, itup))
102  {
103  /* We shouldn't be here since we're inserting to the empty page */
104  elog(ERROR, "could not add new bloom tuple to empty page");
105  }
106 
107  /* Next item was added successfully */
108  buildstate->count++;
109  }
110 
111  /* Update total tuple count */
112  buildstate->indtuples += 1;
113 
114  MemoryContextSwitchTo(oldCtx);
115  MemoryContextReset(buildstate->tmpCtx);
116 }
117 
118 /*
119  * Build a new bloom index.
120  */
123 {
124  IndexBuildResult *result;
125  double reltuples;
126  BloomBuildState buildstate;
127 
128  if (RelationGetNumberOfBlocks(index) != 0)
129  elog(ERROR, "index \"%s\" already contains data",
130  RelationGetRelationName(index));
131 
132  /* Initialize the meta page */
133  BloomInitMetapage(index);
134 
135  /* Initialize the bloom build state */
136  memset(&buildstate, 0, sizeof(buildstate));
137  initBloomState(&buildstate.blstate, index);
139  "Bloom build temporary context",
141  initCachedPage(&buildstate);
142 
143  /* Do the heap scan */
144  reltuples = IndexBuildHeapScan(heap, index, indexInfo, true,
145  bloomBuildCallback, (void *) &buildstate,
146  NULL);
147 
148  /* Flush last page if needed (it will be, unless heap was empty) */
149  if (buildstate.count > 0)
150  flushCachedPage(index, &buildstate);
151 
152  MemoryContextDelete(buildstate.tmpCtx);
153 
154  result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));
155  result->heap_tuples = reltuples;
156  result->index_tuples = buildstate.indtuples;
157 
158  return result;
159 }
160 
161 /*
162  * Build an empty bloom index in the initialization fork.
163  */
164 void
166 {
167  Page metapage;
168 
169  /* Construct metapage. */
170  metapage = (Page) palloc(BLCKSZ);
171  BloomFillMetapage(index, metapage);
172 
173  /*
174  * Write the page and log it. It might seem that an immediate sync would
175  * be sufficient to guarantee that the file exists on disk, but recovery
176  * itself might remove it while replaying, for example, an
177  * XLOG_DBASE_CREATE or XLOG_TBLSPC_CREATE record. Therefore, we need
178  * this even when wal_level=minimal.
179  */
182  (char *) metapage, true);
184  BLOOM_METAPAGE_BLKNO, metapage, true);
185 
186  /*
187  * An immediate sync is required even if we xlog'd the page, because the
188  * write did not go through shared_buffers and therefore a concurrent
189  * checkpoint may have moved the redo pointer past our xlog record.
190  */
192 }
193 
194 /*
195  * Insert new tuple to the bloom index.
196  */
197 bool
199  ItemPointer ht_ctid, Relation heapRel,
200  IndexUniqueCheck checkUnique,
201  IndexInfo *indexInfo)
202 {
203  BloomState blstate;
204  BloomTuple *itup;
205  MemoryContext oldCtx;
206  MemoryContext insertCtx;
207  BloomMetaPageData *metaData;
208  Buffer buffer,
209  metaBuffer;
210  Page page,
211  metaPage;
213  OffsetNumber nStart;
215 
217  "Bloom insert temporary context",
219 
220  oldCtx = MemoryContextSwitchTo(insertCtx);
221 
222  initBloomState(&blstate, index);
223  itup = BloomFormTuple(&blstate, ht_ctid, values, isnull);
224 
225  /*
226  * At first, try to insert new tuple to the first page in notFullPage
227  * array. If successful, we don't need to modify the meta page.
228  */
229  metaBuffer = ReadBuffer(index, BLOOM_METAPAGE_BLKNO);
230  LockBuffer(metaBuffer, BUFFER_LOCK_SHARE);
231  metaData = BloomPageGetMeta(BufferGetPage(metaBuffer));
232 
233  if (metaData->nEnd > metaData->nStart)
234  {
235  Page page;
236 
237  blkno = metaData->notFullPage[metaData->nStart];
238  Assert(blkno != InvalidBlockNumber);
239 
240  /* Don't hold metabuffer lock while doing insert */
241  LockBuffer(metaBuffer, BUFFER_LOCK_UNLOCK);
242 
243  buffer = ReadBuffer(index, blkno);
245 
246  state = GenericXLogStart(index);
247  page = GenericXLogRegisterBuffer(state, buffer, 0);
248 
249  /*
250  * We might have found a page that was recently deleted by VACUUM. If
251  * so, we can reuse it, but we must reinitialize it.
252  */
253  if (PageIsNew(page) || BloomPageIsDeleted(page))
254  BloomInitPage(page, 0);
255 
256  if (BloomPageAddItem(&blstate, page, itup))
257  {
258  /* Success! Apply the change, clean up, and exit */
259  GenericXLogFinish(state);
260  UnlockReleaseBuffer(buffer);
261  ReleaseBuffer(metaBuffer);
262  MemoryContextSwitchTo(oldCtx);
263  MemoryContextDelete(insertCtx);
264  return false;
265  }
266 
267  /* Didn't fit, must try other pages */
268  GenericXLogAbort(state);
269  UnlockReleaseBuffer(buffer);
270  }
271  else
272  {
273  /* No entries in notFullPage */
274  LockBuffer(metaBuffer, BUFFER_LOCK_UNLOCK);
275  }
276 
277  /*
278  * Try other pages in notFullPage array. We will have to change nStart in
279  * metapage. Thus, grab exclusive lock on metapage.
280  */
281  LockBuffer(metaBuffer, BUFFER_LOCK_EXCLUSIVE);
282 
283  /* nStart might have changed while we didn't have lock */
284  nStart = metaData->nStart;
285 
286  /* Skip first page if we already tried it above */
287  if (nStart < metaData->nEnd &&
288  blkno == metaData->notFullPage[nStart])
289  nStart++;
290 
291  /*
292  * This loop iterates for each page we try from the notFullPage array, and
293  * will also initialize a GenericXLogState for the fallback case of having
294  * to allocate a new page.
295  */
296  for (;;)
297  {
298  state = GenericXLogStart(index);
299 
300  /* get modifiable copy of metapage */
301  metaPage = GenericXLogRegisterBuffer(state, metaBuffer, 0);
302  metaData = BloomPageGetMeta(metaPage);
303 
304  if (nStart >= metaData->nEnd)
305  break; /* no more entries in notFullPage array */
306 
307  blkno = metaData->notFullPage[nStart];
308  Assert(blkno != InvalidBlockNumber);
309 
310  buffer = ReadBuffer(index, blkno);
312  page = GenericXLogRegisterBuffer(state, buffer, 0);
313 
314  /* Basically same logic as above */
315  if (PageIsNew(page) || BloomPageIsDeleted(page))
316  BloomInitPage(page, 0);
317 
318  if (BloomPageAddItem(&blstate, page, itup))
319  {
320  /* Success! Apply the changes, clean up, and exit */
321  metaData->nStart = nStart;
322  GenericXLogFinish(state);
323  UnlockReleaseBuffer(buffer);
324  UnlockReleaseBuffer(metaBuffer);
325  MemoryContextSwitchTo(oldCtx);
326  MemoryContextDelete(insertCtx);
327  return false;
328  }
329 
330  /* Didn't fit, must try other pages */
331  GenericXLogAbort(state);
332  UnlockReleaseBuffer(buffer);
333  nStart++;
334  }
335 
336  /*
337  * Didn't find place to insert in notFullPage array. Allocate new page.
338  * (XXX is it good to do this while holding ex-lock on the metapage??)
339  */
340  buffer = BloomNewBuffer(index);
341 
342  page = GenericXLogRegisterBuffer(state, buffer, GENERIC_XLOG_FULL_IMAGE);
343  BloomInitPage(page, 0);
344 
345  if (!BloomPageAddItem(&blstate, page, itup))
346  {
347  /* We shouldn't be here since we're inserting to an empty page */
348  elog(ERROR, "could not add new bloom tuple to empty page");
349  }
350 
351  /* Reset notFullPage array to contain just this new page */
352  metaData->nStart = 0;
353  metaData->nEnd = 1;
354  metaData->notFullPage[0] = BufferGetBlockNumber(buffer);
355 
356  /* Apply the changes, clean up, and exit */
357  GenericXLogFinish(state);
358 
359  UnlockReleaseBuffer(buffer);
360  UnlockReleaseBuffer(metaBuffer);
361 
362  MemoryContextSwitchTo(oldCtx);
363  MemoryContextDelete(insertCtx);
364 
365  return false;
366 }
#define BLOOM_METAPAGE_BLKNO
Definition: bloom.h:77
#define BUFFER_LOCK_UNLOCK
Definition: bufmgr.h:87
#define BloomPageIsDeleted(page)
Definition: bloom.h:63
static void bloomBuildCallback(Relation index, HeapTuple htup, Datum *values, bool *isnull, bool tupleIsAlive, void *state)
Definition: blinsert.c:75
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:211
IndexBuildResult * blbuild(Relation heap, Relation index, IndexInfo *indexInfo)
Definition: blinsert.c:122
void initBloomState(BloomState *state, Relation index)
Definition: blutils.c:154
uint16 nEnd
Definition: bloom.h:124
Buffer BloomNewBuffer(Relation index)
Definition: blutils.c:345
char data[BLCKSZ]
Definition: blinsert.c:39
struct SMgrRelationData * rd_smgr
Definition: rel.h:57
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:136
uint32 BlockNumber
Definition: block.h:31
void ReleaseBuffer(Buffer buffer)
Definition: bufmgr.c:3309
#define BUFFER_LOCK_EXCLUSIVE
Definition: bufmgr.h:89
Page GenericXLogRegisterBuffer(GenericXLogState *state, Buffer buffer, int flags)
Definition: generic_xlog.c:298
uint16 OffsetNumber
Definition: off.h:24
Definition: type.h:89
IndexUniqueCheck
Definition: genam.h:111
void UnlockReleaseBuffer(Buffer buffer)
Definition: bufmgr.c:3332
#define ERROR
Definition: elog.h:43
void BloomInitPage(Page page, uint16 flags)
Definition: blutils.c:399
static void flushCachedPage(Relation index, BloomBuildState *buildstate)
Definition: blinsert.c:47
double IndexBuildHeapScan(Relation heapRelation, Relation indexRelation, IndexInfo *indexInfo, bool allow_sync, IndexBuildCallback callback, void *callback_state, HeapScanDesc scan)
Definition: index.c:2418
ItemPointerData t_self
Definition: htup.h:65
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:192
RelFileNodeBackend smgr_rnode
Definition: smgr.h:43
void smgrwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool skipFsync)
Definition: smgr.c:647
MemoryContext tmpCtx
Definition: blinsert.c:37
bool BloomPageAddItem(BloomState *state, Page page, BloomTuple *tuple)
Definition: blutils.c:310
#define RelationGetRelationName(relation)
Definition: rel.h:441
MemoryContext CurrentMemoryContext
Definition: mcxt.c:38
FreeBlockNumberArray notFullPage
Definition: bloom.h:126
#define BufferGetPage(buffer)
Definition: bufmgr.h:160
#define BloomPageGetMeta(page)
Definition: bloom.h:135
#define AllocSetContextCreate(parent, name, allocparams)
Definition: memutils.h:170
uintptr_t Datum
Definition: postgres.h:367
void LockBuffer(Buffer buffer, int mode)
Definition: bufmgr.c:3546
#define RelationGetNumberOfBlocks(reln)
Definition: bufmgr.h:199
static void initCachedPage(BloomBuildState *buildstate)
Definition: blinsert.c:64
RelFileNode node
Definition: relfilenode.h:74
#define Assert(condition)
Definition: c.h:699
void BloomFillMetapage(Relation index, Page metaPage)
Definition: blutils.c:415
Definition: regguts.h:298
BloomTuple * BloomFormTuple(BloomState *state, ItemPointer iptr, Datum *values, bool *isnull)
Definition: blutils.c:285
XLogRecPtr GenericXLogFinish(GenericXLogState *state)
Definition: generic_xlog.c:336
WalTimeSample buffer[LAG_TRACKER_BUFFER_SIZE]
Definition: walsender.c:215
Buffer ReadBuffer(Relation reln, BlockNumber blockNum)
Definition: bufmgr.c:594
bool blinsert(Relation index, Datum *values, bool *isnull, ItemPointer ht_ctid, Relation heapRel, IndexUniqueCheck checkUnique, IndexInfo *indexInfo)
Definition: blinsert.c:198
#define InvalidBlockNumber
Definition: block.h:33
void PageSetChecksumInplace(Page page, BlockNumber blkno)
Definition: bufpage.c:1195
void blbuildempty(Relation index)
Definition: blinsert.c:165
uint16 nStart
Definition: bloom.h:123
static Datum values[MAXATTR]
Definition: bootstrap.c:164
BlockNumber BufferGetBlockNumber(Buffer buffer)
Definition: bufmgr.c:2605
#define PageIsNew(page)
Definition: bufpage.h:225
void * palloc(Size size)
Definition: mcxt.c:924
XLogRecPtr log_newpage(RelFileNode *rnode, ForkNumber forkNum, BlockNumber blkno, Page page, bool page_std)
Definition: xloginsert.c:972
BloomState blstate
Definition: blinsert.c:35
#define BUFFER_LOCK_SHARE
Definition: bufmgr.h:88
PG_MODULE_MAGIC
Definition: blinsert.c:27
void GenericXLogAbort(GenericXLogState *state)
Definition: generic_xlog.c:447
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:98
#define elog
Definition: elog.h:219
int64 indtuples
Definition: blinsert.c:36
int Buffer
Definition: buf.h:23
void smgrimmedsync(SMgrRelation reln, ForkNumber forknum)
Definition: smgr.c:734
#define GENERIC_XLOG_FULL_IMAGE
Definition: generic_xlog.h:26
float4 reltuples
Definition: pg_class.h:44
GenericXLogState * GenericXLogStart(Relation relation)
Definition: generic_xlog.c:270
Pointer Page
Definition: bufpage.h:74
double index_tuples
Definition: genam.h:33
double heap_tuples
Definition: genam.h:32
void BloomInitMetapage(Relation index)
Definition: blutils.c:447