PostgreSQL Source Code  git master
blinsert.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * blinsert.c
4  * Bloom index build and insert functions.
5  *
6  * Copyright (c) 2016-2019, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * contrib/bloom/blinsert.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 #include "postgres.h"
14 
15 #include "access/genam.h"
16 #include "access/generic_xlog.h"
17 #include "access/tableam.h"
18 #include "catalog/index.h"
19 #include "miscadmin.h"
20 #include "storage/bufmgr.h"
21 #include "storage/indexfsm.h"
22 #include "storage/smgr.h"
23 #include "utils/memutils.h"
24 #include "utils/rel.h"
25 
26 #include "bloom.h"
27 
29 
30 /*
31  * State of bloom index build. We accumulate one page data here before
32  * flushing it to buffer manager.
33  */
34 typedef struct
35 {
36  BloomState blstate; /* bloom index state */
37  int64 indtuples; /* total number of tuples indexed */
38  MemoryContext tmpCtx; /* temporary memory context reset after each
39  * tuple */
40  PGAlignedBlock data; /* cached page */
41  int count; /* number of tuples in cached page */
43 
44 /*
45  * Flush page cached in BloomBuildState.
46  */
47 static void
49 {
50  Page page;
51  Buffer buffer = BloomNewBuffer(index);
53 
54  state = GenericXLogStart(index);
56  memcpy(page, buildstate->data.data, BLCKSZ);
57  GenericXLogFinish(state);
58  UnlockReleaseBuffer(buffer);
59 }
60 
61 /*
62  * (Re)initialize cached page in BloomBuildState.
63  */
64 static void
66 {
67  memset(buildstate->data.data, 0, BLCKSZ);
68  BloomInitPage(buildstate->data.data, 0);
69  buildstate->count = 0;
70 }
71 
72 /*
73  * Per-tuple callback for table_index_build_scan.
74  */
75 static void
77  bool *isnull, bool tupleIsAlive, void *state)
78 {
79  BloomBuildState *buildstate = (BloomBuildState *) state;
80  MemoryContext oldCtx;
81  BloomTuple *itup;
82 
83  oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
84 
85  itup = BloomFormTuple(&buildstate->blstate, &htup->t_self, values, isnull);
86 
87  /* Try to add next item to cached page */
88  if (BloomPageAddItem(&buildstate->blstate, buildstate->data.data, itup))
89  {
90  /* Next item was added successfully */
91  buildstate->count++;
92  }
93  else
94  {
95  /* Cached page is full, flush it out and make a new one */
96  flushCachedPage(index, buildstate);
97 
99 
100  initCachedPage(buildstate);
101 
102  if (!BloomPageAddItem(&buildstate->blstate, buildstate->data.data, itup))
103  {
104  /* We shouldn't be here since we're inserting to the empty page */
105  elog(ERROR, "could not add new bloom tuple to empty page");
106  }
107 
108  /* Next item was added successfully */
109  buildstate->count++;
110  }
111 
112  /* Update total tuple count */
113  buildstate->indtuples += 1;
114 
115  MemoryContextSwitchTo(oldCtx);
116  MemoryContextReset(buildstate->tmpCtx);
117 }
118 
119 /*
120  * Build a new bloom index.
121  */
124 {
125  IndexBuildResult *result;
126  double reltuples;
127  BloomBuildState buildstate;
128 
129  if (RelationGetNumberOfBlocks(index) != 0)
130  elog(ERROR, "index \"%s\" already contains data",
131  RelationGetRelationName(index));
132 
133  /* Initialize the meta page */
134  BloomInitMetapage(index);
135 
136  /* Initialize the bloom build state */
137  memset(&buildstate, 0, sizeof(buildstate));
138  initBloomState(&buildstate.blstate, index);
140  "Bloom build temporary context",
142  initCachedPage(&buildstate);
143 
144  /* Do the heap scan */
145  reltuples = table_index_build_scan(heap, index, indexInfo, true, true,
146  bloomBuildCallback, (void *) &buildstate,
147  NULL);
148 
149  /* Flush last page if needed (it will be, unless heap was empty) */
150  if (buildstate.count > 0)
151  flushCachedPage(index, &buildstate);
152 
153  MemoryContextDelete(buildstate.tmpCtx);
154 
155  result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));
156  result->heap_tuples = reltuples;
157  result->index_tuples = buildstate.indtuples;
158 
159  return result;
160 }
161 
162 /*
163  * Build an empty bloom index in the initialization fork.
164  */
165 void
167 {
168  Page metapage;
169 
170  /* Construct metapage. */
171  metapage = (Page) palloc(BLCKSZ);
172  BloomFillMetapage(index, metapage);
173 
174  /*
175  * Write the page and log it. It might seem that an immediate sync would
176  * be sufficient to guarantee that the file exists on disk, but recovery
177  * itself might remove it while replaying, for example, an
178  * XLOG_DBASE_CREATE or XLOG_TBLSPC_CREATE record. Therefore, we need
179  * this even when wal_level=minimal.
180  */
183  (char *) metapage, true);
185  BLOOM_METAPAGE_BLKNO, metapage, true);
186 
187  /*
188  * An immediate sync is required even if we xlog'd the page, because the
189  * write did not go through shared_buffers and therefore a concurrent
190  * checkpoint may have moved the redo pointer past our xlog record.
191  */
193 }
194 
195 /*
196  * Insert new tuple to the bloom index.
197  */
198 bool
200  ItemPointer ht_ctid, Relation heapRel,
201  IndexUniqueCheck checkUnique,
202  IndexInfo *indexInfo)
203 {
204  BloomState blstate;
205  BloomTuple *itup;
206  MemoryContext oldCtx;
207  MemoryContext insertCtx;
208  BloomMetaPageData *metaData;
209  Buffer buffer,
210  metaBuffer;
211  Page page,
212  metaPage;
214  OffsetNumber nStart;
216 
218  "Bloom insert temporary context",
220 
221  oldCtx = MemoryContextSwitchTo(insertCtx);
222 
223  initBloomState(&blstate, index);
224  itup = BloomFormTuple(&blstate, ht_ctid, values, isnull);
225 
226  /*
227  * At first, try to insert new tuple to the first page in notFullPage
228  * array. If successful, we don't need to modify the meta page.
229  */
230  metaBuffer = ReadBuffer(index, BLOOM_METAPAGE_BLKNO);
231  LockBuffer(metaBuffer, BUFFER_LOCK_SHARE);
232  metaData = BloomPageGetMeta(BufferGetPage(metaBuffer));
233 
234  if (metaData->nEnd > metaData->nStart)
235  {
236  Page page;
237 
238  blkno = metaData->notFullPage[metaData->nStart];
239  Assert(blkno != InvalidBlockNumber);
240 
241  /* Don't hold metabuffer lock while doing insert */
242  LockBuffer(metaBuffer, BUFFER_LOCK_UNLOCK);
243 
244  buffer = ReadBuffer(index, blkno);
246 
247  state = GenericXLogStart(index);
248  page = GenericXLogRegisterBuffer(state, buffer, 0);
249 
250  /*
251  * We might have found a page that was recently deleted by VACUUM. If
252  * so, we can reuse it, but we must reinitialize it.
253  */
254  if (PageIsNew(page) || BloomPageIsDeleted(page))
255  BloomInitPage(page, 0);
256 
257  if (BloomPageAddItem(&blstate, page, itup))
258  {
259  /* Success! Apply the change, clean up, and exit */
260  GenericXLogFinish(state);
261  UnlockReleaseBuffer(buffer);
262  ReleaseBuffer(metaBuffer);
263  MemoryContextSwitchTo(oldCtx);
264  MemoryContextDelete(insertCtx);
265  return false;
266  }
267 
268  /* Didn't fit, must try other pages */
269  GenericXLogAbort(state);
270  UnlockReleaseBuffer(buffer);
271  }
272  else
273  {
274  /* No entries in notFullPage */
275  LockBuffer(metaBuffer, BUFFER_LOCK_UNLOCK);
276  }
277 
278  /*
279  * Try other pages in notFullPage array. We will have to change nStart in
280  * metapage. Thus, grab exclusive lock on metapage.
281  */
282  LockBuffer(metaBuffer, BUFFER_LOCK_EXCLUSIVE);
283 
284  /* nStart might have changed while we didn't have lock */
285  nStart = metaData->nStart;
286 
287  /* Skip first page if we already tried it above */
288  if (nStart < metaData->nEnd &&
289  blkno == metaData->notFullPage[nStart])
290  nStart++;
291 
292  /*
293  * This loop iterates for each page we try from the notFullPage array, and
294  * will also initialize a GenericXLogState for the fallback case of having
295  * to allocate a new page.
296  */
297  for (;;)
298  {
299  state = GenericXLogStart(index);
300 
301  /* get modifiable copy of metapage */
302  metaPage = GenericXLogRegisterBuffer(state, metaBuffer, 0);
303  metaData = BloomPageGetMeta(metaPage);
304 
305  if (nStart >= metaData->nEnd)
306  break; /* no more entries in notFullPage array */
307 
308  blkno = metaData->notFullPage[nStart];
309  Assert(blkno != InvalidBlockNumber);
310 
311  buffer = ReadBuffer(index, blkno);
313  page = GenericXLogRegisterBuffer(state, buffer, 0);
314 
315  /* Basically same logic as above */
316  if (PageIsNew(page) || BloomPageIsDeleted(page))
317  BloomInitPage(page, 0);
318 
319  if (BloomPageAddItem(&blstate, page, itup))
320  {
321  /* Success! Apply the changes, clean up, and exit */
322  metaData->nStart = nStart;
323  GenericXLogFinish(state);
324  UnlockReleaseBuffer(buffer);
325  UnlockReleaseBuffer(metaBuffer);
326  MemoryContextSwitchTo(oldCtx);
327  MemoryContextDelete(insertCtx);
328  return false;
329  }
330 
331  /* Didn't fit, must try other pages */
332  GenericXLogAbort(state);
333  UnlockReleaseBuffer(buffer);
334  nStart++;
335  }
336 
337  /*
338  * Didn't find place to insert in notFullPage array. Allocate new page.
339  * (XXX is it good to do this while holding ex-lock on the metapage??)
340  */
341  buffer = BloomNewBuffer(index);
342 
343  page = GenericXLogRegisterBuffer(state, buffer, GENERIC_XLOG_FULL_IMAGE);
344  BloomInitPage(page, 0);
345 
346  if (!BloomPageAddItem(&blstate, page, itup))
347  {
348  /* We shouldn't be here since we're inserting to an empty page */
349  elog(ERROR, "could not add new bloom tuple to empty page");
350  }
351 
352  /* Reset notFullPage array to contain just this new page */
353  metaData->nStart = 0;
354  metaData->nEnd = 1;
355  metaData->notFullPage[0] = BufferGetBlockNumber(buffer);
356 
357  /* Apply the changes, clean up, and exit */
358  GenericXLogFinish(state);
359 
360  UnlockReleaseBuffer(buffer);
361  UnlockReleaseBuffer(metaBuffer);
362 
363  MemoryContextSwitchTo(oldCtx);
364  MemoryContextDelete(insertCtx);
365 
366  return false;
367 }
#define BLOOM_METAPAGE_BLKNO
Definition: bloom.h:77
#define BUFFER_LOCK_UNLOCK
Definition: bufmgr.h:86
#define BloomPageIsDeleted(page)
Definition: bloom.h:63
static void bloomBuildCallback(Relation index, HeapTuple htup, Datum *values, bool *isnull, bool tupleIsAlive, void *state)
Definition: blinsert.c:76
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:211
#define AllocSetContextCreate
Definition: memutils.h:170
IndexBuildResult * blbuild(Relation heap, Relation index, IndexInfo *indexInfo)
Definition: blinsert.c:123
PGAlignedBlock data
Definition: blinsert.c:40
void initBloomState(BloomState *state, Relation index)
Definition: blutils.c:157
uint16 nEnd
Definition: bloom.h:124
Buffer BloomNewBuffer(Relation index)
Definition: blutils.c:349
struct SMgrRelationData * rd_smgr
Definition: rel.h:56
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:136
uint32 BlockNumber
Definition: block.h:31
void ReleaseBuffer(Buffer buffer)
Definition: bufmgr.c:3365
#define BUFFER_LOCK_EXCLUSIVE
Definition: bufmgr.h:88
Page GenericXLogRegisterBuffer(GenericXLogState *state, Buffer buffer, int flags)
Definition: generic_xlog.c:295
uint16 OffsetNumber
Definition: off.h:24
Definition: type.h:89
IndexUniqueCheck
Definition: genam.h:112
char data[BLCKSZ]
Definition: c.h:1060
void UnlockReleaseBuffer(Buffer buffer)
Definition: bufmgr.c:3388
#define ERROR
Definition: elog.h:43
void BloomInitPage(Page page, uint16 flags)
Definition: blutils.c:403
static void flushCachedPage(Relation index, BloomBuildState *buildstate)
Definition: blinsert.c:48
static double table_index_build_scan(Relation table_rel, Relation index_rel, struct IndexInfo *index_info, bool allow_sync, bool progress, IndexBuildCallback callback, void *callback_state, TableScanDesc scan)
Definition: tableam.h:1499
ItemPointerData t_self
Definition: htup.h:65
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:192
RelFileNodeBackend smgr_rnode
Definition: smgr.h:42
void smgrwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool skipFsync)
Definition: smgr.c:530
MemoryContext tmpCtx
Definition: blinsert.c:38
bool BloomPageAddItem(BloomState *state, Page page, BloomTuple *tuple)
Definition: blutils.c:314
#define RelationGetRelationName(relation)
Definition: rel.h:453
MemoryContext CurrentMemoryContext
Definition: mcxt.c:38
FreeBlockNumberArray notFullPage
Definition: bloom.h:126
#define BufferGetPage(buffer)
Definition: bufmgr.h:159
#define BloomPageGetMeta(page)
Definition: bloom.h:135
uintptr_t Datum
Definition: postgres.h:367
void LockBuffer(Buffer buffer, int mode)
Definition: bufmgr.c:3602
#define RelationGetNumberOfBlocks(reln)
Definition: bufmgr.h:198
static void initCachedPage(BloomBuildState *buildstate)
Definition: blinsert.c:65
RelFileNode node
Definition: relfilenode.h:74
#define Assert(condition)
Definition: c.h:732
void BloomFillMetapage(Relation index, Page metaPage)
Definition: blutils.c:419
Definition: regguts.h:298
BloomTuple * BloomFormTuple(BloomState *state, ItemPointer iptr, Datum *values, bool *isnull)
Definition: blutils.c:289
XLogRecPtr GenericXLogFinish(GenericXLogState *state)
Definition: generic_xlog.c:333
Buffer ReadBuffer(Relation reln, BlockNumber blockNum)
Definition: bufmgr.c:596
bool blinsert(Relation index, Datum *values, bool *isnull, ItemPointer ht_ctid, Relation heapRel, IndexUniqueCheck checkUnique, IndexInfo *indexInfo)
Definition: blinsert.c:199
#define InvalidBlockNumber
Definition: block.h:33
void PageSetChecksumInplace(Page page, BlockNumber blkno)
Definition: bufpage.c:1198
void blbuildempty(Relation index)
Definition: blinsert.c:166
uint16 nStart
Definition: bloom.h:123
static Datum values[MAXATTR]
Definition: bootstrap.c:167
BlockNumber BufferGetBlockNumber(Buffer buffer)
Definition: bufmgr.c:2613
#define PageIsNew(page)
Definition: bufpage.h:229
void * palloc(Size size)
Definition: mcxt.c:949
XLogRecPtr log_newpage(RelFileNode *rnode, ForkNumber forkNum, BlockNumber blkno, Page page, bool page_std)
Definition: xloginsert.c:972
#define elog(elevel,...)
Definition: elog.h:226
BloomState blstate
Definition: blinsert.c:36
#define BUFFER_LOCK_SHARE
Definition: bufmgr.h:87
PG_MODULE_MAGIC
Definition: blinsert.c:28
void GenericXLogAbort(GenericXLogState *state)
Definition: generic_xlog.c:444
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:99
int64 indtuples
Definition: blinsert.c:37
int Buffer
Definition: buf.h:23
void smgrimmedsync(SMgrRelation reln, ForkNumber forknum)
Definition: smgr.c:637
#define GENERIC_XLOG_FULL_IMAGE
Definition: generic_xlog.h:26
float4 reltuples
Definition: pg_class.h:63
GenericXLogState * GenericXLogStart(Relation relation)
Definition: generic_xlog.c:267
Pointer Page
Definition: bufpage.h:78
double index_tuples
Definition: genam.h:33
double heap_tuples
Definition: genam.h:32
void BloomInitMetapage(Relation index)
Definition: blutils.c:451