PostgreSQL Source Code  git master
storage.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * storage.c
4  * code to create and destroy physical storage for relations
5  *
6  * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  *
10  * IDENTIFICATION
11  * src/backend/catalog/storage.c
12  *
13  * NOTES
14  * Some of this code used to be in storage/smgr/smgr.c, and the
15  * function names still reflect that.
16  *
17  *-------------------------------------------------------------------------
18  */
19 
20 #include "postgres.h"
21 
22 #include "access/parallel.h"
23 #include "access/visibilitymap.h"
24 #include "access/xact.h"
25 #include "access/xlog.h"
26 #include "access/xloginsert.h"
27 #include "access/xlogutils.h"
28 #include "catalog/storage.h"
29 #include "catalog/storage_xlog.h"
30 #include "miscadmin.h"
31 #include "storage/freespace.h"
32 #include "storage/smgr.h"
33 #include "utils/hsearch.h"
34 #include "utils/memutils.h"
35 #include "utils/rel.h"
36 
37 /* GUC variables */
38 int wal_skip_threshold = 2048; /* in kilobytes */
39 
40 /*
41  * We keep a list of all relations (represented as RelFileNode values)
42  * that have been created or deleted in the current transaction. When
43  * a relation is created, we create the physical file immediately, but
44  * remember it so that we can delete the file again if the current
45  * transaction is aborted. Conversely, a deletion request is NOT
46  * executed immediately, but is just entered in the list. When and if
47  * the transaction commits, we can delete the physical file.
48  *
49  * To handle subtransactions, every entry is marked with its transaction
50  * nesting level. At subtransaction commit, we reassign the subtransaction's
51  * entries to the parent nesting level. At subtransaction abort, we can
52  * immediately execute the abort-time actions for all entries of the current
53  * nesting level.
54  *
55  * NOTE: the list is kept in TopMemoryContext to be sure it won't disappear
56  * unbetimes. It'd probably be OK to keep it in TopTransactionContext,
57  * but I'm being paranoid.
58  */
59 
60 typedef struct PendingRelDelete
61 {
62  RelFileNode relnode; /* relation that may need to be deleted */
63  BackendId backend; /* InvalidBackendId if not a temp rel */
64  bool atCommit; /* T=delete at commit; F=delete at abort */
65  int nestLevel; /* xact nesting level of request */
66  struct PendingRelDelete *next; /* linked-list link */
68 
69 typedef struct PendingRelSync
70 {
72  bool is_truncated; /* Has the file experienced truncation? */
74 
75 static PendingRelDelete *pendingDeletes = NULL; /* head of linked list */
77 
78 
79 /*
80  * AddPendingSync
81  * Queue an at-commit fsync.
82  */
83 static void
85 {
86  PendingRelSync *pending;
87  bool found;
88 
89  /* create the hash if not yet */
90  if (!pendingSyncHash)
91  {
92  HASHCTL ctl;
93 
94  ctl.keysize = sizeof(RelFileNode);
95  ctl.entrysize = sizeof(PendingRelSync);
97  pendingSyncHash = hash_create("pending sync hash", 16, &ctl,
99  }
100 
101  pending = hash_search(pendingSyncHash, rnode, HASH_ENTER, &found);
102  Assert(!found);
103  pending->is_truncated = false;
104 }
105 
106 /*
107  * RelationCreateStorage
108  * Create physical storage for a relation.
109  *
110  * Create the underlying disk file storage for the relation. This only
111  * creates the main fork; additional forks are created lazily by the
112  * modules that need them.
113  *
114  * This function is transactional. The creation is WAL-logged, and if the
115  * transaction aborts later on, the storage will be destroyed.
116  */
118 RelationCreateStorage(RelFileNode rnode, char relpersistence)
119 {
120  PendingRelDelete *pending;
121  SMgrRelation srel;
123  bool needs_wal;
124 
125  Assert(!IsInParallelMode()); /* couldn't update pendingSyncHash */
126 
127  switch (relpersistence)
128  {
129  case RELPERSISTENCE_TEMP:
130  backend = BackendIdForTempRelations();
131  needs_wal = false;
132  break;
133  case RELPERSISTENCE_UNLOGGED:
134  backend = InvalidBackendId;
135  needs_wal = false;
136  break;
137  case RELPERSISTENCE_PERMANENT:
138  backend = InvalidBackendId;
139  needs_wal = true;
140  break;
141  default:
142  elog(ERROR, "invalid relpersistence: %c", relpersistence);
143  return NULL; /* placate compiler */
144  }
145 
146  srel = smgropen(rnode, backend);
147  smgrcreate(srel, MAIN_FORKNUM, false);
148 
149  if (needs_wal)
151 
152  /* Add the relation to the list of stuff to delete at abort */
153  pending = (PendingRelDelete *)
155  pending->relnode = rnode;
156  pending->backend = backend;
157  pending->atCommit = false; /* delete if abort */
159  pending->next = pendingDeletes;
160  pendingDeletes = pending;
161 
162  if (relpersistence == RELPERSISTENCE_PERMANENT && !XLogIsNeeded())
163  {
164  Assert(backend == InvalidBackendId);
165  AddPendingSync(&rnode);
166  }
167 
168  return srel;
169 }
170 
171 /*
172  * Perform XLogInsert of an XLOG_SMGR_CREATE record to WAL.
173  */
174 void
175 log_smgrcreate(const RelFileNode *rnode, ForkNumber forkNum)
176 {
177  xl_smgr_create xlrec;
178 
179  /*
180  * Make an XLOG entry reporting the file creation.
181  */
182  xlrec.rnode = *rnode;
183  xlrec.forkNum = forkNum;
184 
185  XLogBeginInsert();
186  XLogRegisterData((char *) &xlrec, sizeof(xlrec));
188 }
189 
190 /*
191  * RelationDropStorage
192  * Schedule unlinking of physical storage at transaction commit.
193  */
194 void
196 {
197  PendingRelDelete *pending;
198 
199  /* Add the relation to the list of stuff to delete at commit */
200  pending = (PendingRelDelete *)
202  pending->relnode = rel->rd_node;
203  pending->backend = rel->rd_backend;
204  pending->atCommit = true; /* delete if commit */
206  pending->next = pendingDeletes;
207  pendingDeletes = pending;
208 
209  /*
210  * NOTE: if the relation was created in this transaction, it will now be
211  * present in the pending-delete list twice, once with atCommit true and
212  * once with atCommit false. Hence, it will be physically deleted at end
213  * of xact in either case (and the other entry will be ignored by
214  * smgrDoPendingDeletes, so no error will occur). We could instead remove
215  * the existing list entry and delete the physical file immediately, but
216  * for now I'll keep the logic simple.
217  */
218 
219  RelationCloseSmgr(rel);
220 }
221 
222 /*
223  * RelationPreserveStorage
224  * Mark a relation as not to be deleted after all.
225  *
226  * We need this function because relation mapping changes are committed
227  * separately from commit of the whole transaction, so it's still possible
228  * for the transaction to abort after the mapping update is done.
229  * When a new physical relation is installed in the map, it would be
230  * scheduled for delete-on-abort, so we'd delete it, and be in trouble.
231  * The relation mapper fixes this by telling us to not delete such relations
232  * after all as part of its commit.
233  *
234  * We also use this to reuse an old build of an index during ALTER TABLE, this
235  * time removing the delete-at-commit entry.
236  *
237  * No-op if the relation is not among those scheduled for deletion.
238  */
239 void
241 {
242  PendingRelDelete *pending;
243  PendingRelDelete *prev;
245 
246  prev = NULL;
247  for (pending = pendingDeletes; pending != NULL; pending = next)
248  {
249  next = pending->next;
250  if (RelFileNodeEquals(rnode, pending->relnode)
251  && pending->atCommit == atCommit)
252  {
253  /* unlink and delete list entry */
254  if (prev)
255  prev->next = next;
256  else
257  pendingDeletes = next;
258  pfree(pending);
259  /* prev does not change */
260  }
261  else
262  {
263  /* unrelated entry, don't touch it */
264  prev = pending;
265  }
266  }
267 }
268 
269 /*
270  * RelationTruncate
271  * Physically truncate a relation to the specified number of blocks.
272  *
273  * This includes getting rid of any buffers for the blocks that are to be
274  * dropped.
275  */
276 void
278 {
279  bool fsm;
280  bool vm;
281  bool need_fsm_vacuum = false;
282  ForkNumber forks[MAX_FORKNUM];
283  BlockNumber blocks[MAX_FORKNUM];
284  int nforks = 0;
285  SMgrRelation reln;
286 
287  /*
288  * Make sure smgr_targblock etc aren't pointing somewhere past new end.
289  * (Note: don't rely on this reln pointer below this loop.)
290  */
291  reln = RelationGetSmgr(rel);
293  for (int i = 0; i <= MAX_FORKNUM; ++i)
295 
296  /* Prepare for truncation of MAIN fork of the relation */
297  forks[nforks] = MAIN_FORKNUM;
298  blocks[nforks] = nblocks;
299  nforks++;
300 
301  /* Prepare for truncation of the FSM if it exists */
303  if (fsm)
304  {
305  blocks[nforks] = FreeSpaceMapPrepareTruncateRel(rel, nblocks);
306  if (BlockNumberIsValid(blocks[nforks]))
307  {
308  forks[nforks] = FSM_FORKNUM;
309  nforks++;
310  need_fsm_vacuum = true;
311  }
312  }
313 
314  /* Prepare for truncation of the visibility map too if it exists */
316  if (vm)
317  {
318  blocks[nforks] = visibilitymap_prepare_truncate(rel, nblocks);
319  if (BlockNumberIsValid(blocks[nforks]))
320  {
321  forks[nforks] = VISIBILITYMAP_FORKNUM;
322  nforks++;
323  }
324  }
325 
326  RelationPreTruncate(rel);
327 
328  /*
329  * We WAL-log the truncation before actually truncating, which means
330  * trouble if the truncation fails. If we then crash, the WAL replay
331  * likely isn't going to succeed in the truncation either, and cause a
332  * PANIC. It's tempting to put a critical section here, but that cure
333  * would be worse than the disease. It would turn a usually harmless
334  * failure to truncate, that might spell trouble at WAL replay, into a
335  * certain PANIC.
336  */
337  if (RelationNeedsWAL(rel))
338  {
339  /*
340  * Make an XLOG entry reporting the file truncation.
341  */
342  XLogRecPtr lsn;
343  xl_smgr_truncate xlrec;
344 
345  xlrec.blkno = nblocks;
346  xlrec.rnode = rel->rd_node;
347  xlrec.flags = SMGR_TRUNCATE_ALL;
348 
349  XLogBeginInsert();
350  XLogRegisterData((char *) &xlrec, sizeof(xlrec));
351 
352  lsn = XLogInsert(RM_SMGR_ID,
354 
355  /*
356  * Flush, because otherwise the truncation of the main relation might
357  * hit the disk before the WAL record, and the truncation of the FSM
358  * or visibility map. If we crashed during that window, we'd be left
359  * with a truncated heap, but the FSM or visibility map would still
360  * contain entries for the non-existent heap pages.
361  */
362  if (fsm || vm)
363  XLogFlush(lsn);
364  }
365 
366  /* Do the real work to truncate relation forks */
367  smgrtruncate(RelationGetSmgr(rel), forks, nforks, blocks);
368 
369  /*
370  * Update upper-level FSM pages to account for the truncation. This is
371  * important because the just-truncated pages were likely marked as
372  * all-free, and would be preferentially selected.
373  */
374  if (need_fsm_vacuum)
376 }
377 
378 /*
379  * RelationPreTruncate
380  * Perform AM-independent work before a physical truncation.
381  *
382  * If an access method's relation_nontransactional_truncate does not call
383  * RelationTruncate(), it must call this before decreasing the table size.
384  */
385 void
387 {
388  PendingRelSync *pending;
389 
390  if (!pendingSyncHash)
391  return;
392 
393  pending = hash_search(pendingSyncHash,
394  &(RelationGetSmgr(rel)->smgr_rnode.node),
395  HASH_FIND, NULL);
396  if (pending)
397  pending->is_truncated = true;
398 }
399 
400 /*
401  * Copy a fork's data, block by block.
402  *
403  * Note that this requires that there is no dirty data in shared buffers. If
404  * it's possible that there are, callers need to flush those using
405  * e.g. FlushRelationBuffers(rel).
406  *
407  * Also note that this is frequently called via locutions such as
408  * RelationCopyStorage(RelationGetSmgr(rel), ...);
409  * That's safe only because we perform only smgr and WAL operations here.
410  * If we invoked anything else, a relcache flush could cause our SMgrRelation
411  * argument to become a dangling pointer.
412  */
413 void
415  ForkNumber forkNum, char relpersistence)
416 {
418  Page page;
419  bool use_wal;
420  bool copying_initfork;
421  BlockNumber nblocks;
422  BlockNumber blkno;
423 
424  page = (Page) buf.data;
425 
426  /*
427  * The init fork for an unlogged relation in many respects has to be
428  * treated the same as normal relation, changes need to be WAL logged and
429  * it needs to be synced to disk.
430  */
431  copying_initfork = relpersistence == RELPERSISTENCE_UNLOGGED &&
432  forkNum == INIT_FORKNUM;
433 
434  /*
435  * We need to log the copied data in WAL iff WAL archiving/streaming is
436  * enabled AND it's a permanent relation. This gives the same answer as
437  * "RelationNeedsWAL(rel) || copying_initfork", because we know the
438  * current operation created a new relfilenode.
439  */
440  use_wal = XLogIsNeeded() &&
441  (relpersistence == RELPERSISTENCE_PERMANENT || copying_initfork);
442 
443  nblocks = smgrnblocks(src, forkNum);
444 
445  for (blkno = 0; blkno < nblocks; blkno++)
446  {
447  /* If we got a cancel signal during the copy of the data, quit */
449 
450  smgrread(src, forkNum, blkno, buf.data);
451 
452  if (!PageIsVerifiedExtended(page, blkno,
454  {
455  /*
456  * For paranoia's sake, capture the file path before invoking the
457  * ereport machinery. This guards against the possibility of a
458  * relcache flush caused by, e.g., an errcontext callback.
459  * (errcontext callbacks shouldn't be risking any such thing, but
460  * people have been known to forget that rule.)
461  */
462  char *relpath = relpathbackend(src->smgr_rnode.node,
463  src->smgr_rnode.backend,
464  forkNum);
465 
466  ereport(ERROR,
468  errmsg("invalid page in block %u of relation %s",
469  blkno, relpath)));
470  }
471 
472  /*
473  * WAL-log the copied page. Unfortunately we don't know what kind of a
474  * page this is, so we have to log the full page including any unused
475  * space.
476  */
477  if (use_wal)
478  log_newpage(&dst->smgr_rnode.node, forkNum, blkno, page, false);
479 
480  PageSetChecksumInplace(page, blkno);
481 
482  /*
483  * Now write the page. We say skipFsync = true because there's no
484  * need for smgr to schedule an fsync for this write; we'll do it
485  * ourselves below.
486  */
487  smgrextend(dst, forkNum, blkno, buf.data, true);
488  }
489 
490  /*
491  * When we WAL-logged rel pages, we must nonetheless fsync them. The
492  * reason is that since we're copying outside shared buffers, a CHECKPOINT
493  * occurring during the copy has no way to flush the previously written
494  * data to disk (indeed it won't know the new rel even exists). A crash
495  * later on would replay WAL from the checkpoint, therefore it wouldn't
496  * replay our earlier WAL entries. If we do not fsync those pages here,
497  * they might still not be on disk when the crash occurs.
498  */
499  if (use_wal || copying_initfork)
500  smgrimmedsync(dst, forkNum);
501 }
502 
503 /*
504  * RelFileNodeSkippingWAL
505  * Check if a BM_PERMANENT relfilenode is using WAL.
506  *
507  * Changes of certain relfilenodes must not write WAL; see "Skipping WAL for
508  * New RelFileNode" in src/backend/access/transam/README. Though it is known
509  * from Relation efficiently, this function is intended for the code paths not
510  * having access to Relation.
511  */
512 bool
514 {
515  if (!pendingSyncHash ||
516  hash_search(pendingSyncHash, &rnode, HASH_FIND, NULL) == NULL)
517  return false;
518 
519  return true;
520 }
521 
522 /*
523  * EstimatePendingSyncsSpace
524  * Estimate space needed to pass syncs to parallel workers.
525  */
526 Size
528 {
529  long entries;
530 
531  entries = pendingSyncHash ? hash_get_num_entries(pendingSyncHash) : 0;
532  return mul_size(1 + entries, sizeof(RelFileNode));
533 }
534 
535 /*
536  * SerializePendingSyncs
537  * Serialize syncs for parallel workers.
538  */
539 void
540 SerializePendingSyncs(Size maxSize, char *startAddress)
541 {
542  HTAB *tmphash;
543  HASHCTL ctl;
544  HASH_SEQ_STATUS scan;
545  PendingRelSync *sync;
546  PendingRelDelete *delete;
547  RelFileNode *src;
548  RelFileNode *dest = (RelFileNode *) startAddress;
549 
550  if (!pendingSyncHash)
551  goto terminate;
552 
553  /* Create temporary hash to collect active relfilenodes */
554  ctl.keysize = sizeof(RelFileNode);
555  ctl.entrysize = sizeof(RelFileNode);
557  tmphash = hash_create("tmp relfilenodes",
558  hash_get_num_entries(pendingSyncHash), &ctl,
560 
561  /* collect all rnodes from pending syncs */
562  hash_seq_init(&scan, pendingSyncHash);
563  while ((sync = (PendingRelSync *) hash_seq_search(&scan)))
564  (void) hash_search(tmphash, &sync->rnode, HASH_ENTER, NULL);
565 
566  /* remove deleted rnodes */
567  for (delete = pendingDeletes; delete != NULL; delete = delete->next)
568  if (delete->atCommit)
569  (void) hash_search(tmphash, (void *) &delete->relnode,
570  HASH_REMOVE, NULL);
571 
572  hash_seq_init(&scan, tmphash);
573  while ((src = (RelFileNode *) hash_seq_search(&scan)))
574  *dest++ = *src;
575 
576  hash_destroy(tmphash);
577 
578 terminate:
579  MemSet(dest, 0, sizeof(RelFileNode));
580 }
581 
582 /*
583  * RestorePendingSyncs
584  * Restore syncs within a parallel worker.
585  *
586  * RelationNeedsWAL() and RelFileNodeSkippingWAL() must offer the correct
587  * answer to parallel workers. Only smgrDoPendingSyncs() reads the
588  * is_truncated field, at end of transaction. Hence, don't restore it.
589  */
590 void
591 RestorePendingSyncs(char *startAddress)
592 {
593  RelFileNode *rnode;
594 
595  Assert(pendingSyncHash == NULL);
596  for (rnode = (RelFileNode *) startAddress; rnode->relNode != 0; rnode++)
597  AddPendingSync(rnode);
598 }
599 
600 /*
601  * smgrDoPendingDeletes() -- Take care of relation deletes at end of xact.
602  *
603  * This also runs when aborting a subxact; we want to clean up a failed
604  * subxact immediately.
605  *
606  * Note: It's possible that we're being asked to remove a relation that has
607  * no physical storage in any fork. In particular, it's possible that we're
608  * cleaning up an old temporary relation for which RemovePgTempFiles has
609  * already recovered the physical storage.
610  */
611 void
612 smgrDoPendingDeletes(bool isCommit)
613 {
615  PendingRelDelete *pending;
616  PendingRelDelete *prev;
618  int nrels = 0,
619  maxrels = 0;
620  SMgrRelation *srels = NULL;
621 
622  prev = NULL;
623  for (pending = pendingDeletes; pending != NULL; pending = next)
624  {
625  next = pending->next;
626  if (pending->nestLevel < nestLevel)
627  {
628  /* outer-level entries should not be processed yet */
629  prev = pending;
630  }
631  else
632  {
633  /* unlink list entry first, so we don't retry on failure */
634  if (prev)
635  prev->next = next;
636  else
637  pendingDeletes = next;
638  /* do deletion if called for */
639  if (pending->atCommit == isCommit)
640  {
641  SMgrRelation srel;
642 
643  srel = smgropen(pending->relnode, pending->backend);
644 
645  /* allocate the initial array, or extend it, if needed */
646  if (maxrels == 0)
647  {
648  maxrels = 8;
649  srels = palloc(sizeof(SMgrRelation) * maxrels);
650  }
651  else if (maxrels <= nrels)
652  {
653  maxrels *= 2;
654  srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
655  }
656 
657  srels[nrels++] = srel;
658  }
659  /* must explicitly free the list entry */
660  pfree(pending);
661  /* prev does not change */
662  }
663  }
664 
665  if (nrels > 0)
666  {
667  smgrdounlinkall(srels, nrels, false);
668 
669  for (int i = 0; i < nrels; i++)
670  smgrclose(srels[i]);
671 
672  pfree(srels);
673  }
674 }
675 
676 /*
677  * smgrDoPendingSyncs() -- Take care of relation syncs at end of xact.
678  */
679 void
680 smgrDoPendingSyncs(bool isCommit, bool isParallelWorker)
681 {
682  PendingRelDelete *pending;
683  int nrels = 0,
684  maxrels = 0;
685  SMgrRelation *srels = NULL;
686  HASH_SEQ_STATUS scan;
687  PendingRelSync *pendingsync;
688 
690 
691  if (!pendingSyncHash)
692  return; /* no relation needs sync */
693 
694  /* Abort -- just throw away all pending syncs */
695  if (!isCommit)
696  {
697  pendingSyncHash = NULL;
698  return;
699  }
700 
702 
703  /* Parallel worker -- just throw away all pending syncs */
704  if (isParallelWorker)
705  {
706  pendingSyncHash = NULL;
707  return;
708  }
709 
710  /* Skip syncing nodes that smgrDoPendingDeletes() will delete. */
711  for (pending = pendingDeletes; pending != NULL; pending = pending->next)
712  if (pending->atCommit)
713  (void) hash_search(pendingSyncHash, (void *) &pending->relnode,
714  HASH_REMOVE, NULL);
715 
716  hash_seq_init(&scan, pendingSyncHash);
717  while ((pendingsync = (PendingRelSync *) hash_seq_search(&scan)))
718  {
719  ForkNumber fork;
720  BlockNumber nblocks[MAX_FORKNUM + 1];
721  BlockNumber total_blocks = 0;
722  SMgrRelation srel;
723 
724  srel = smgropen(pendingsync->rnode, InvalidBackendId);
725 
726  /*
727  * We emit newpage WAL records for smaller relations.
728  *
729  * Small WAL records have a chance to be emitted along with other
730  * backends' WAL records. We emit WAL records instead of syncing for
731  * files that are smaller than a certain threshold, expecting faster
732  * commit. The threshold is defined by the GUC wal_skip_threshold.
733  */
734  if (!pendingsync->is_truncated)
735  {
736  for (fork = 0; fork <= MAX_FORKNUM; fork++)
737  {
738  if (smgrexists(srel, fork))
739  {
740  BlockNumber n = smgrnblocks(srel, fork);
741 
742  /* we shouldn't come here for unlogged relations */
743  Assert(fork != INIT_FORKNUM);
744  nblocks[fork] = n;
745  total_blocks += n;
746  }
747  else
748  nblocks[fork] = InvalidBlockNumber;
749  }
750  }
751 
752  /*
753  * Sync file or emit WAL records for its contents.
754  *
755  * Although we emit WAL record if the file is small enough, do file
756  * sync regardless of the size if the file has experienced a
757  * truncation. It is because the file would be followed by trailing
758  * garbage blocks after a crash recovery if, while a past longer file
759  * had been flushed out, we omitted syncing-out of the file and
760  * emitted WAL instead. You might think that we could choose WAL if
761  * the current main fork is longer than ever, but there's a case where
762  * main fork is longer than ever but FSM fork gets shorter.
763  */
764  if (pendingsync->is_truncated ||
765  total_blocks * BLCKSZ / 1024 >= wal_skip_threshold)
766  {
767  /* allocate the initial array, or extend it, if needed */
768  if (maxrels == 0)
769  {
770  maxrels = 8;
771  srels = palloc(sizeof(SMgrRelation) * maxrels);
772  }
773  else if (maxrels <= nrels)
774  {
775  maxrels *= 2;
776  srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
777  }
778 
779  srels[nrels++] = srel;
780  }
781  else
782  {
783  /* Emit WAL records for all blocks. The file is small enough. */
784  for (fork = 0; fork <= MAX_FORKNUM; fork++)
785  {
786  int n = nblocks[fork];
787  Relation rel;
788 
789  if (!BlockNumberIsValid(n))
790  continue;
791 
792  /*
793  * Emit WAL for the whole file. Unfortunately we don't know
794  * what kind of a page this is, so we have to log the full
795  * page including any unused space. ReadBufferExtended()
796  * counts some pgstat events; unfortunately, we discard them.
797  */
799  log_newpage_range(rel, fork, 0, n, false);
801  }
802  }
803  }
804 
805  pendingSyncHash = NULL;
806 
807  if (nrels > 0)
808  {
809  smgrdosyncall(srels, nrels);
810  pfree(srels);
811  }
812 }
813 
814 /*
815  * smgrGetPendingDeletes() -- Get a list of non-temp relations to be deleted.
816  *
817  * The return value is the number of relations scheduled for termination.
818  * *ptr is set to point to a freshly-palloc'd array of RelFileNodes.
819  * If there are no relations to be deleted, *ptr is set to NULL.
820  *
821  * Only non-temporary relations are included in the returned list. This is OK
822  * because the list is used only in contexts where temporary relations don't
823  * matter: we're either writing to the two-phase state file (and transactions
824  * that have touched temp tables can't be prepared) or we're writing to xlog
825  * (and all temporary files will be zapped if we restart anyway, so no need
826  * for redo to do it also).
827  *
828  * Note that the list does not include anything scheduled for termination
829  * by upper-level transactions.
830  */
831 int
832 smgrGetPendingDeletes(bool forCommit, RelFileNode **ptr)
833 {
835  int nrels;
836  RelFileNode *rptr;
837  PendingRelDelete *pending;
838 
839  nrels = 0;
840  for (pending = pendingDeletes; pending != NULL; pending = pending->next)
841  {
842  if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
843  && pending->backend == InvalidBackendId)
844  nrels++;
845  }
846  if (nrels == 0)
847  {
848  *ptr = NULL;
849  return 0;
850  }
851  rptr = (RelFileNode *) palloc(nrels * sizeof(RelFileNode));
852  *ptr = rptr;
853  for (pending = pendingDeletes; pending != NULL; pending = pending->next)
854  {
855  if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
856  && pending->backend == InvalidBackendId)
857  {
858  *rptr = pending->relnode;
859  rptr++;
860  }
861  }
862  return nrels;
863 }
864 
865 /*
866  * PostPrepare_smgr -- Clean up after a successful PREPARE
867  *
868  * What we have to do here is throw away the in-memory state about pending
869  * relation deletes. It's all been recorded in the 2PC state file and
870  * it's no longer smgr's job to worry about it.
871  */
872 void
874 {
875  PendingRelDelete *pending;
877 
878  for (pending = pendingDeletes; pending != NULL; pending = next)
879  {
880  next = pending->next;
881  pendingDeletes = next;
882  /* must explicitly free the list entry */
883  pfree(pending);
884  }
885 }
886 
887 
888 /*
889  * AtSubCommit_smgr() --- Take care of subtransaction commit.
890  *
891  * Reassign all items in the pending-deletes list to the parent transaction.
892  */
893 void
895 {
897  PendingRelDelete *pending;
898 
899  for (pending = pendingDeletes; pending != NULL; pending = pending->next)
900  {
901  if (pending->nestLevel >= nestLevel)
902  pending->nestLevel = nestLevel - 1;
903  }
904 }
905 
906 /*
907  * AtSubAbort_smgr() --- Take care of subtransaction abort.
908  *
909  * Delete created relations and forget about deleted relations.
910  * We can execute these operations immediately because we know this
911  * subtransaction will not commit.
912  */
913 void
915 {
916  smgrDoPendingDeletes(false);
917 }
918 
919 void
921 {
922  XLogRecPtr lsn = record->EndRecPtr;
923  uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
924 
925  /* Backup blocks are not used in smgr records */
926  Assert(!XLogRecHasAnyBlockRefs(record));
927 
928  if (info == XLOG_SMGR_CREATE)
929  {
930  xl_smgr_create *xlrec = (xl_smgr_create *) XLogRecGetData(record);
931  SMgrRelation reln;
932 
933  reln = smgropen(xlrec->rnode, InvalidBackendId);
934  smgrcreate(reln, xlrec->forkNum, true);
935  }
936  else if (info == XLOG_SMGR_TRUNCATE)
937  {
938  xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
939  SMgrRelation reln;
940  Relation rel;
941  ForkNumber forks[MAX_FORKNUM];
942  BlockNumber blocks[MAX_FORKNUM];
943  int nforks = 0;
944  bool need_fsm_vacuum = false;
945 
946  reln = smgropen(xlrec->rnode, InvalidBackendId);
947 
948  /*
949  * Forcibly create relation if it doesn't exist (which suggests that
950  * it was dropped somewhere later in the WAL sequence). As in
951  * XLogReadBufferForRedo, we prefer to recreate the rel and replay the
952  * log as best we can until the drop is seen.
953  */
954  smgrcreate(reln, MAIN_FORKNUM, true);
955 
956  /*
957  * Before we perform the truncation, update minimum recovery point to
958  * cover this WAL record. Once the relation is truncated, there's no
959  * going back. The buffer manager enforces the WAL-first rule for
960  * normal updates to relation files, so that the minimum recovery
961  * point is always updated before the corresponding change in the data
962  * file is flushed to disk. We have to do the same manually here.
963  *
964  * Doing this before the truncation means that if the truncation fails
965  * for some reason, you cannot start up the system even after restart,
966  * until you fix the underlying situation so that the truncation will
967  * succeed. Alternatively, we could update the minimum recovery point
968  * after truncation, but that would leave a small window where the
969  * WAL-first rule could be violated.
970  */
971  XLogFlush(lsn);
972 
973  /* Prepare for truncation of MAIN fork */
974  if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
975  {
976  forks[nforks] = MAIN_FORKNUM;
977  blocks[nforks] = xlrec->blkno;
978  nforks++;
979 
980  /* Also tell xlogutils.c about it */
981  XLogTruncateRelation(xlrec->rnode, MAIN_FORKNUM, xlrec->blkno);
982  }
983 
984  /* Prepare for truncation of FSM and VM too */
985  rel = CreateFakeRelcacheEntry(xlrec->rnode);
986 
987  if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
988  smgrexists(reln, FSM_FORKNUM))
989  {
990  blocks[nforks] = FreeSpaceMapPrepareTruncateRel(rel, xlrec->blkno);
991  if (BlockNumberIsValid(blocks[nforks]))
992  {
993  forks[nforks] = FSM_FORKNUM;
994  nforks++;
995  need_fsm_vacuum = true;
996  }
997  }
998  if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
1000  {
1001  blocks[nforks] = visibilitymap_prepare_truncate(rel, xlrec->blkno);
1002  if (BlockNumberIsValid(blocks[nforks]))
1003  {
1004  forks[nforks] = VISIBILITYMAP_FORKNUM;
1005  nforks++;
1006  }
1007  }
1008 
1009  /* Do the real work to truncate relation forks */
1010  if (nforks > 0)
1011  smgrtruncate(reln, forks, nforks, blocks);
1012 
1013  /*
1014  * Update upper-level FSM pages to account for the truncation. This is
1015  * important because the just-truncated pages were likely marked as
1016  * all-free, and would be preferentially selected.
1017  */
1018  if (need_fsm_vacuum)
1019  FreeSpaceMapVacuumRange(rel, xlrec->blkno,
1021 
1022  FreeFakeRelcacheEntry(rel);
1023  }
1024  else
1025  elog(PANIC, "smgr_redo: unknown op code %u", info);
1026 }
void RelationPreserveStorage(RelFileNode rnode, bool atCommit)
Definition: storage.c:240
void XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum, BlockNumber nblocks)
Definition: xlogutils.c:642
void smgrDoPendingDeletes(bool isCommit)
Definition: storage.c:612
void hash_destroy(HTAB *hashp)
Definition: dynahash.c:862
BackendId backend
Definition: storage.c:63
void smgrclose(SMgrRelation reln)
Definition: smgr.c:256
#define SMGR_TRUNCATE_HEAP
Definition: storage_xlog.h:40
void smgrcreate(SMgrRelation reln, ForkNumber forknum, bool isRedo)
Definition: smgr.c:333
MemoryContext TopTransactionContext
Definition: mcxt.c:53
#define XLR_SPECIAL_REL_UPDATE
Definition: xlogrecord.h:71
#define HASH_CONTEXT
Definition: hsearch.h:102
#define HASH_ELEM
Definition: hsearch.h:95
void RelationPreTruncate(Relation rel)
Definition: storage.c:386
MemoryContext hcxt
Definition: hsearch.h:86
struct PendingRelSync PendingRelSync
struct PendingRelDelete PendingRelDelete
bool is_truncated
Definition: storage.c:72
#define XLogIsNeeded()
Definition: xlog.h:208
unsigned char uint8
Definition: c.h:439
#define SMGR_TRUNCATE_FSM
Definition: storage_xlog.h:42
void AtSubCommit_smgr(void)
Definition: storage.c:894
Size entrysize
Definition: hsearch.h:76
int errcode(int sqlerrcode)
Definition: elog.c:698
void smgrdounlinkall(SMgrRelation *rels, int nrels, bool isRedo)
Definition: smgr.c:384
RelFileNode rnode
Definition: storage.c:71
#define MemSet(start, val, len)
Definition: c.h:1008
#define RelationCloseSmgr(relation)
Definition: rel.h:557
long hash_get_num_entries(HTAB *hashp)
Definition: dynahash.c:1382
uint32 BlockNumber
Definition: block.h:31
RelFileNode rnode
Definition: storage_xlog.h:49
bool smgrexists(SMgrRelation reln, ForkNumber forknum)
Definition: smgr.c:247
void * hash_search(HTAB *hashp, const void *keyPtr, HASHACTION action, bool *foundPtr)
Definition: dynahash.c:954
#define PANIC
Definition: elog.h:50
void XLogFlush(XLogRecPtr record)
Definition: xlog.c:2881
BlockNumber smgr_cached_nblocks[MAX_FORKNUM+1]
Definition: smgr.h:54
#define XLOG_SMGR_CREATE
Definition: storage_xlog.h:30
XLogRecPtr EndRecPtr
Definition: xlogreader.h:176
void smgrread(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer)
Definition: smgr.c:501
void RestorePendingSyncs(char *startAddress)
Definition: storage.c:591
static void AddPendingSync(const RelFileNode *rnode)
Definition: storage.c:84
Definition: dynahash.c:219
char data[BLCKSZ]
Definition: c.h:1141
void pfree(void *pointer)
Definition: mcxt.c:1169
SMgrRelation RelationCreateStorage(RelFileNode rnode, char relpersistence)
Definition: storage.c:118
bool IsInParallelMode(void)
Definition: xact.c:1012
#define XLogRecGetData(decoder)
Definition: xlogreader.h:310
HTAB * pendingSyncHash
Definition: storage.c:76
#define ERROR
Definition: elog.h:46
#define PIV_LOG_WARNING
Definition: bufpage.h:413
void smgr_redo(XLogReaderState *record)
Definition: storage.c:920
#define AssertPendingSyncs_RelationCache()
Definition: relcache.h:133
RelFileNodeBackend smgr_rnode
Definition: smgr.h:42
int smgrGetPendingDeletes(bool forCommit, RelFileNode **ptr)
Definition: storage.c:832
Relation CreateFakeRelcacheEntry(RelFileNode rnode)
Definition: xlogutils.c:554
#define BackendIdForTempRelations()
Definition: backendid.h:34
struct RelFileNode RelFileNode
static char * buf
Definition: pg_test_fsync.c:68
BlockNumber FreeSpaceMapPrepareTruncateRel(Relation rel, BlockNumber nblocks)
Definition: freespace.c:261
#define PIV_REPORT_STAT
Definition: bufpage.h:414
HTAB * hash_create(const char *tabname, long nelem, const HASHCTL *info, int flags)
Definition: dynahash.c:349
MemoryContext CurrentMemoryContext
Definition: mcxt.c:42
bool PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags)
Definition: bufpage.c:88
#define XLogRecGetInfo(decoder)
Definition: xlogreader.h:305
MemoryContext TopMemoryContext
Definition: mcxt.c:48
SMgrRelation smgropen(RelFileNode rnode, BackendId backend)
Definition: smgr.c:146
ForkNumber
Definition: relpath.h:40
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
#define XLOG_SMGR_TRUNCATE
Definition: storage_xlog.h:31
void FreeFakeRelcacheEntry(Relation fakerel)
Definition: xlogutils.c:597
struct PendingRelDelete * next
Definition: storage.c:66
void XLogRegisterData(char *data, int len)
Definition: xloginsert.c:340
XLogRecPtr XLogInsert(RmgrId rmid, uint8 info)
Definition: xloginsert.c:432
#define HASH_BLOBS
Definition: hsearch.h:97
Size mul_size(Size s1, Size s2)
Definition: shmem.c:519
#define InvalidBackendId
Definition: backendid.h:23
void RelationDropStorage(Relation rel)
Definition: storage.c:195
Size EstimatePendingSyncsSpace(void)
Definition: storage.c:527
void PostPrepare_smgr(void)
Definition: storage.c:873
int BackendId
Definition: backendid.h:21
void SerializePendingSyncs(Size maxSize, char *startAddress)
Definition: storage.c:540
Size keysize
Definition: hsearch.h:75
void smgrtruncate(SMgrRelation reln, ForkNumber *forknum, int nforks, BlockNumber *nblocks)
Definition: smgr.c:595
void RelationCopyStorage(SMgrRelation src, SMgrRelation dst, ForkNumber forkNum, char relpersistence)
Definition: storage.c:414
#define ereport(elevel,...)
Definition: elog.h:157
#define BlockNumberIsValid(blockNumber)
Definition: block.h:70
RelFileNode node
Definition: relfilenode.h:74
int GetCurrentTransactionNestLevel(void)
Definition: xact.c:857
RelFileNode rd_node
Definition: rel.h:56
ForkNumber forkNum
Definition: storage_xlog.h:36
BlockNumber smgrnblocks(SMgrRelation reln, ForkNumber forknum)
Definition: smgr.c:548
uint64 XLogRecPtr
Definition: xlogdefs.h:21
BackendId rd_backend
Definition: rel.h:59
#define Assert(condition)
Definition: c.h:804
#define XLR_INFO_MASK
Definition: xlogrecord.h:62
int wal_skip_threshold
Definition: storage.c:38
BlockNumber smgr_targblock
Definition: smgr.h:53
static SMgrRelation RelationGetSmgr(Relation rel)
Definition: rel.h:544
void AtSubAbort_smgr(void)
Definition: storage.c:914
size_t Size
Definition: c.h:540
BackendId backend
Definition: relfilenode.h:75
#define InvalidBlockNumber
Definition: block.h:33
#define SMGR_TRUNCATE_VM
Definition: storage_xlog.h:41
#define MAX_FORKNUM
Definition: relpath.h:55
void PageSetChecksumInplace(Page page, BlockNumber blkno)
Definition: bufpage.c:1532
void * hash_seq_search(HASH_SEQ_STATUS *status)
Definition: dynahash.c:1436
#define RelationNeedsWAL(relation)
Definition: rel.h:601
bool RelFileNodeSkippingWAL(RelFileNode rnode)
Definition: storage.c:513
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1182
void hash_seq_init(HASH_SEQ_STATUS *status, HTAB *hashp)
Definition: dynahash.c:1426
void smgrextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer, bool skipFsync)
Definition: smgr.c:462
void log_newpage_range(Relation rel, ForkNumber forkNum, BlockNumber startblk, BlockNumber endblk, bool page_std)
Definition: xloginsert.c:1175
static PendingRelDelete * pendingDeletes
Definition: storage.c:75
void * palloc(Size size)
Definition: mcxt.c:1062
int errmsg(const char *fmt,...)
Definition: elog.c:909
RelFileNode relnode
Definition: storage.c:62
void * MemoryContextAlloc(MemoryContext context, Size size)
Definition: mcxt.c:863
XLogRecPtr log_newpage(RelFileNode *rnode, ForkNumber forkNum, BlockNumber blkno, Page page, bool page_std)
Definition: xloginsert.c:1048
#define elog(elevel,...)
Definition: elog.h:232
int i
#define relpath(rnode, forknum)
Definition: relpath.h:87
BlockNumber visibilitymap_prepare_truncate(Relation rel, BlockNumber nheapblocks)
#define XLogRecHasAnyBlockRefs(decoder)
Definition: xlogreader.h:312
BlockNumber blkno
Definition: storage_xlog.h:48
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:120
void smgrDoPendingSyncs(bool isCommit, bool isParallelWorker)
Definition: storage.c:680
#define SMGR_TRUNCATE_ALL
Definition: storage_xlog.h:43
void XLogBeginInsert(void)
Definition: xloginsert.c:135
#define relpathbackend(rnode, backend, forknum)
Definition: relpath.h:78
void smgrimmedsync(SMgrRelation reln, ForkNumber forknum)
Definition: smgr.c:660
void FreeSpaceMapVacuumRange(Relation rel, BlockNumber start, BlockNumber end)
Definition: freespace.c:352
RelFileNode rnode
Definition: storage_xlog.h:35
Pointer Page
Definition: bufpage.h:78
#define RelFileNodeEquals(node1, node2)
Definition: relfilenode.h:88
dlist_node node
Definition: smgr.h:72
void RelationTruncate(Relation rel, BlockNumber nblocks)
Definition: storage.c:277
void log_smgrcreate(const RelFileNode *rnode, ForkNumber forkNum)
Definition: storage.c:175
void smgrdosyncall(SMgrRelation *rels, int nrels)
Definition: smgr.c:348