PostgreSQL Source Code  git master
walsummarizer.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * walsummarizer.c
4  *
5  * Background process to perform WAL summarization, if it is enabled.
6  * It continuously scans the write-ahead log and periodically emits a
7  * summary file which indicates which blocks in which relation forks
8  * were modified by WAL records in the LSN range covered by the summary
9  * file. See walsummary.c and blkreftable.c for more details on the
10  * naming and contents of WAL summary files.
11  *
12  * If configured to do, this background process will also remove WAL
13  * summary files when the file timestamp is older than a configurable
14  * threshold (but only if the WAL has been removed first).
15  *
16  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
17  *
18  * IDENTIFICATION
19  * src/backend/postmaster/walsummarizer.c
20  *
21  *-------------------------------------------------------------------------
22  */
23 #include "postgres.h"
24 
25 #include "access/timeline.h"
26 #include "access/xlog.h"
27 #include "access/xlog_internal.h"
28 #include "access/xlogrecovery.h"
29 #include "access/xlogutils.h"
30 #include "backup/walsummary.h"
31 #include "catalog/storage_xlog.h"
33 #include "common/blkreftable.h"
34 #include "libpq/pqsignal.h"
35 #include "miscadmin.h"
36 #include "postmaster/auxprocess.h"
37 #include "postmaster/interrupt.h"
40 #include "storage/fd.h"
41 #include "storage/ipc.h"
42 #include "storage/latch.h"
43 #include "storage/lwlock.h"
44 #include "storage/proc.h"
45 #include "storage/procsignal.h"
46 #include "storage/shmem.h"
47 #include "utils/guc.h"
48 #include "utils/memutils.h"
49 #include "utils/wait_event.h"
50 
51 /*
52  * Data in shared memory related to WAL summarization.
53  */
54 typedef struct
55 {
56  /*
57  * These fields are protected by WALSummarizerLock.
58  *
59  * Until we've discovered what summary files already exist on disk and
60  * stored that information in shared memory, initialized is false and the
61  * other fields here contain no meaningful information. After that has
62  * been done, initialized is true.
63  *
64  * summarized_tli and summarized_lsn indicate the last LSN and TLI at
65  * which the next summary file will start. Normally, these are the LSN and
66  * TLI at which the last file ended; in such case, lsn_is_exact is true.
67  * If, however, the LSN is just an approximation, then lsn_is_exact is
68  * false. This can happen if, for example, there are no existing WAL
69  * summary files at startup. In that case, we have to derive the position
70  * at which to start summarizing from the WAL files that exist on disk,
71  * and so the LSN might point to the start of the next file even though
72  * that might happen to be in the middle of a WAL record.
73  *
74  * summarizer_pgprocno is the proc number of the summarizer process, if
75  * one is running, or else INVALID_PROC_NUMBER.
76  *
77  * pending_lsn is used by the summarizer to advertise the ending LSN of a
78  * record it has recently read. It shouldn't ever be less than
79  * summarized_lsn, but might be greater, because the summarizer buffers
80  * data for a range of LSNs in memory before writing out a new file.
81  */
88 
89  /*
90  * This field handles its own synchronization.
91  */
94 
95 /*
96  * Private data for our xlogreader's page read callback.
97  */
98 typedef struct
99 {
101  bool historic;
105 
106 /* Pointer to shared memory state. */
108 
109 /*
110  * When we reach end of WAL and need to read more, we sleep for a number of
111  * milliseconds that is an integer multiple of MS_PER_SLEEP_QUANTUM. This is
112  * the multiplier. It should vary between 1 and MAX_SLEEP_QUANTA, depending
113  * on system activity. See summarizer_wait_for_wal() for how we adjust this.
114  */
115 static long sleep_quanta = 1;
116 
117 /*
118  * The sleep time will always be a multiple of 200ms and will not exceed
119  * thirty seconds (150 * 200 = 30 * 1000). Note that the timeout here needs
120  * to be substantially less than the maximum amount of time for which an
121  * incremental backup will wait for this process to catch up. Otherwise, an
122  * incremental backup might time out on an idle system just because we sleep
123  * for too long.
124  */
125 #define MAX_SLEEP_QUANTA 150
126 #define MS_PER_SLEEP_QUANTUM 200
127 
128 /*
129  * This is a count of the number of pages of WAL that we've read since the
130  * last time we waited for more WAL to appear.
131  */
133 
134 /*
135  * Most recent RedoRecPtr value observed by MaybeRemoveOldWalSummaries.
136  */
138 
139 /*
140  * GUC parameters
141  */
142 bool summarize_wal = false;
144 
145 static void WalSummarizerShutdown(int code, Datum arg);
146 static XLogRecPtr GetLatestLSN(TimeLineID *tli);
147 static void HandleWalSummarizerInterrupts(void);
148 static XLogRecPtr SummarizeWAL(TimeLineID tli, XLogRecPtr start_lsn,
149  bool exact, XLogRecPtr switch_lsn,
150  XLogRecPtr maximum_lsn);
152  BlockRefTable *brtab);
154  BlockRefTable *brtab);
156  BlockRefTable *brtab);
159  XLogRecPtr targetPagePtr,
160  int reqLen,
161  XLogRecPtr targetRecPtr,
162  char *cur_page);
163 static void summarizer_wait_for_wal(void);
164 static void MaybeRemoveOldWalSummaries(void);
165 
166 /*
167  * Amount of shared memory required for this module.
168  */
169 Size
171 {
172  return sizeof(WalSummarizerData);
173 }
174 
175 /*
176  * Create or attach to shared memory segment for this module.
177  */
178 void
180 {
181  bool found;
182 
184  ShmemInitStruct("Wal Summarizer Ctl", WalSummarizerShmemSize(),
185  &found);
186 
187  if (!found)
188  {
189  /*
190  * First time through, so initialize.
191  *
192  * We're just filling in dummy values here -- the real initialization
193  * will happen when GetOldestUnsummarizedLSN() is called for the first
194  * time.
195  */
196  WalSummarizerCtl->initialized = false;
203  }
204 }
205 
206 /*
207  * Entry point for walsummarizer process.
208  */
209 void
210 WalSummarizerMain(char *startup_data, size_t startup_data_len)
211 {
212  sigjmp_buf local_sigjmp_buf;
214 
215  /*
216  * Within this function, 'current_lsn' and 'current_tli' refer to the
217  * point from which the next WAL summary file should start. 'exact' is
218  * true if 'current_lsn' is known to be the start of a WAL record or WAL
219  * segment, and false if it might be in the middle of a record someplace.
220  *
221  * 'switch_lsn' and 'switch_tli', if set, are the LSN at which we need to
222  * switch to a new timeline and the timeline to which we need to switch.
223  * If not set, we either haven't figured out the answers yet or we're
224  * already on the latest timeline.
225  */
226  XLogRecPtr current_lsn;
227  TimeLineID current_tli;
228  bool exact;
229  XLogRecPtr switch_lsn = InvalidXLogRecPtr;
230  TimeLineID switch_tli = 0;
231 
232  Assert(startup_data_len == 0);
233 
236 
237  ereport(DEBUG1,
238  (errmsg_internal("WAL summarizer started")));
239 
240  /*
241  * Properly accept or ignore signals the postmaster might send us
242  *
243  * We have no particular use for SIGINT at the moment, but seems
244  * reasonable to treat like SIGTERM.
245  */
249  /* SIGQUIT handler was already set up by InitPostmasterChild */
253  pqsignal(SIGUSR2, SIG_IGN); /* not used */
254 
255  /* Advertise ourselves. */
257  LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
259  LWLockRelease(WALSummarizerLock);
260 
261  /* Create and switch to a memory context that we can reset on error. */
263  "Wal Summarizer",
266 
267  /*
268  * Reset some signals that are accepted by postmaster but not here
269  */
271 
272  /*
273  * If an exception is encountered, processing resumes here.
274  */
275  if (sigsetjmp(local_sigjmp_buf, 1) != 0)
276  {
277  /* Since not using PG_TRY, must reset error stack by hand */
278  error_context_stack = NULL;
279 
280  /* Prevent interrupts while cleaning up */
281  HOLD_INTERRUPTS();
282 
283  /* Report the error to the server log */
284  EmitErrorReport();
285 
286  /* Release resources we might have acquired. */
291  AtEOXact_Files(false);
292  AtEOXact_HashTables(false);
293 
294  /*
295  * Now return to normal top-level context and clear ErrorContext for
296  * next time.
297  */
299  FlushErrorState();
300 
301  /* Flush any leaked data in the top-level context */
303 
304  /* Now we can allow interrupts again */
306 
307  /*
308  * Sleep for 10 seconds before attempting to resume operations in
309  * order to avoid excessive logging.
310  *
311  * Many of the likely error conditions are things that will repeat
312  * every time. For example, if the WAL can't be read or the summary
313  * can't be written, only administrator action will cure the problem.
314  * So a really fast retry time doesn't seem to be especially
315  * beneficial, and it will clutter the logs.
316  */
317  (void) WaitLatch(MyLatch,
319  10000,
320  WAIT_EVENT_WAL_SUMMARIZER_ERROR);
321  }
322 
323  /* We can now handle ereport(ERROR) */
324  PG_exception_stack = &local_sigjmp_buf;
325 
326  /*
327  * Unblock signals (they were blocked when the postmaster forked us)
328  */
329  sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
330 
331  /*
332  * Fetch information about previous progress from shared memory, and ask
333  * GetOldestUnsummarizedLSN to reset pending_lsn to summarized_lsn. We
334  * might be recovering from an error, and if so, pending_lsn might have
335  * advanced past summarized_lsn, but any WAL we read previously has been
336  * lost and will need to be reread.
337  *
338  * If we discover that WAL summarization is not enabled, just exit.
339  */
340  current_lsn = GetOldestUnsummarizedLSN(&current_tli, &exact);
341  if (XLogRecPtrIsInvalid(current_lsn))
342  proc_exit(0);
343 
344  /*
345  * Loop forever
346  */
347  for (;;)
348  {
349  XLogRecPtr latest_lsn;
350  TimeLineID latest_tli;
351  XLogRecPtr end_of_summary_lsn;
352 
353  /* Flush any leaked data in the top-level context */
355 
356  /* Process any signals received recently. */
358 
359  /* If it's time to remove any old WAL summaries, do that now. */
361 
362  /* Find the LSN and TLI up to which we can safely summarize. */
363  latest_lsn = GetLatestLSN(&latest_tli);
364 
365  /*
366  * If we're summarizing a historic timeline and we haven't yet
367  * computed the point at which to switch to the next timeline, do that
368  * now.
369  *
370  * Note that if this is a standby, what was previously the current
371  * timeline could become historic at any time.
372  *
373  * We could try to make this more efficient by caching the results of
374  * readTimeLineHistory when latest_tli has not changed, but since we
375  * only have to do this once per timeline switch, we probably wouldn't
376  * save any significant amount of work in practice.
377  */
378  if (current_tli != latest_tli && XLogRecPtrIsInvalid(switch_lsn))
379  {
380  List *tles = readTimeLineHistory(latest_tli);
381 
382  switch_lsn = tliSwitchPoint(current_tli, tles, &switch_tli);
383  ereport(DEBUG1,
384  errmsg("switch point from TLI %u to TLI %u is at %X/%X",
385  current_tli, switch_tli, LSN_FORMAT_ARGS(switch_lsn)));
386  }
387 
388  /*
389  * If we've reached the switch LSN, we can't summarize anything else
390  * on this timeline. Switch to the next timeline and go around again.
391  */
392  if (!XLogRecPtrIsInvalid(switch_lsn) && current_lsn >= switch_lsn)
393  {
394  current_tli = switch_tli;
395  switch_lsn = InvalidXLogRecPtr;
396  switch_tli = 0;
397  continue;
398  }
399 
400  /* Summarize WAL. */
401  end_of_summary_lsn = SummarizeWAL(current_tli,
402  current_lsn, exact,
403  switch_lsn, latest_lsn);
404  Assert(!XLogRecPtrIsInvalid(end_of_summary_lsn));
405  Assert(end_of_summary_lsn >= current_lsn);
406 
407  /*
408  * Update state for next loop iteration.
409  *
410  * Next summary file should start from exactly where this one ended.
411  */
412  current_lsn = end_of_summary_lsn;
413  exact = true;
414 
415  /* Update state in shared memory. */
416  LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
417  Assert(WalSummarizerCtl->pending_lsn <= end_of_summary_lsn);
418  WalSummarizerCtl->summarized_lsn = end_of_summary_lsn;
419  WalSummarizerCtl->summarized_tli = current_tli;
421  WalSummarizerCtl->pending_lsn = end_of_summary_lsn;
422  LWLockRelease(WALSummarizerLock);
423 
424  /* Wake up anyone waiting for more summary files to be written. */
426  }
427 }
428 
429 /*
430  * Get information about the state of the WAL summarizer.
431  */
432 void
433 GetWalSummarizerState(TimeLineID *summarized_tli, XLogRecPtr *summarized_lsn,
434  XLogRecPtr *pending_lsn, int *summarizer_pid)
435 {
436  LWLockAcquire(WALSummarizerLock, LW_SHARED);
438  {
439  /*
440  * If initialized is false, the rest of the structure contents are
441  * undefined.
442  */
443  *summarized_tli = 0;
444  *summarized_lsn = InvalidXLogRecPtr;
445  *pending_lsn = InvalidXLogRecPtr;
446  *summarizer_pid = -1;
447  }
448  else
449  {
450  int summarizer_pgprocno = WalSummarizerCtl->summarizer_pgprocno;
451 
452  *summarized_tli = WalSummarizerCtl->summarized_tli;
453  *summarized_lsn = WalSummarizerCtl->summarized_lsn;
454  if (summarizer_pgprocno == INVALID_PROC_NUMBER)
455  {
456  /*
457  * If the summarizer has exited, the fact that it had processed
458  * beyond summarized_lsn is irrelevant now.
459  */
460  *pending_lsn = WalSummarizerCtl->summarized_lsn;
461  *summarizer_pid = -1;
462  }
463  else
464  {
465  *pending_lsn = WalSummarizerCtl->pending_lsn;
466 
467  /*
468  * We're not fussed about inexact answers here, since they could
469  * become stale instantly, so we don't bother taking the lock, but
470  * make sure that invalid PID values are normalized to -1.
471  */
472  *summarizer_pid = GetPGProcByNumber(summarizer_pgprocno)->pid;
473  if (*summarizer_pid <= 0)
474  *summarizer_pid = -1;
475  }
476  }
477  LWLockRelease(WALSummarizerLock);
478 }
479 
480 /*
481  * Get the oldest LSN in this server's timeline history that has not yet been
482  * summarized, and update shared memory state as appropriate.
483  *
484  * If *tli != NULL, it will be set to the TLI for the LSN that is returned.
485  *
486  * If *lsn_is_exact != NULL, it will be set to true if the returned LSN is
487  * necessarily the start of a WAL record and false if it's just the beginning
488  * of a WAL segment.
489  */
491 GetOldestUnsummarizedLSN(TimeLineID *tli, bool *lsn_is_exact)
492 {
493  TimeLineID latest_tli;
494  int n;
495  List *tles;
496  XLogRecPtr unsummarized_lsn = InvalidXLogRecPtr;
497  TimeLineID unsummarized_tli = 0;
498  bool should_make_exact = false;
499  List *existing_summaries;
500  ListCell *lc;
501  bool am_wal_summarizer = AmWalSummarizerProcess();
502 
503  /* If not summarizing WAL, do nothing. */
504  if (!summarize_wal)
505  return InvalidXLogRecPtr;
506 
507  /*
508  * If we are not the WAL summarizer process, then we normally just want to
509  * read the values from shared memory. However, as an exception, if shared
510  * memory hasn't been initialized yet, then we need to do that so that we
511  * can read legal values and not remove any WAL too early.
512  */
513  if (!am_wal_summarizer)
514  {
515  LWLockAcquire(WALSummarizerLock, LW_SHARED);
516 
518  {
519  unsummarized_lsn = WalSummarizerCtl->summarized_lsn;
520  if (tli != NULL)
522  if (lsn_is_exact != NULL)
523  *lsn_is_exact = WalSummarizerCtl->lsn_is_exact;
524  LWLockRelease(WALSummarizerLock);
525  return unsummarized_lsn;
526  }
527 
528  LWLockRelease(WALSummarizerLock);
529  }
530 
531  /*
532  * Find the oldest timeline on which WAL still exists, and the earliest
533  * segment for which it exists.
534  *
535  * Note that we do this every time the WAL summarizer process restarts or
536  * recovers from an error, in case the contents of pg_wal have changed
537  * under us e.g. if some files were removed, either manually - which
538  * shouldn't really happen, but might - or by postgres itself, if
539  * summarize_wal was turned off and then back on again.
540  */
541  (void) GetLatestLSN(&latest_tli);
542  tles = readTimeLineHistory(latest_tli);
543  for (n = list_length(tles) - 1; n >= 0; --n)
544  {
545  TimeLineHistoryEntry *tle = list_nth(tles, n);
546  XLogSegNo oldest_segno;
547 
548  oldest_segno = XLogGetOldestSegno(tle->tli);
549  if (oldest_segno != 0)
550  {
551  /* Compute oldest LSN that still exists on disk. */
552  XLogSegNoOffsetToRecPtr(oldest_segno, 0, wal_segment_size,
553  unsummarized_lsn);
554 
555  unsummarized_tli = tle->tli;
556  break;
557  }
558  }
559 
560  /*
561  * Don't try to summarize anything older than the end LSN of the newest
562  * summary file that exists for this timeline.
563  */
564  existing_summaries =
565  GetWalSummaries(unsummarized_tli,
567  foreach(lc, existing_summaries)
568  {
569  WalSummaryFile *ws = lfirst(lc);
570 
571  if (ws->end_lsn > unsummarized_lsn)
572  {
573  unsummarized_lsn = ws->end_lsn;
574  should_make_exact = true;
575  }
576  }
577 
578  /* It really should not be possible for us to find no WAL. */
579  if (unsummarized_tli == 0)
580  ereport(ERROR,
581  errcode(ERRCODE_INTERNAL_ERROR),
582  errmsg_internal("no WAL found on timeline %u", latest_tli));
583 
584  /*
585  * If we're the WAL summarizer, we always want to store the values we just
586  * computed into shared memory, because those are the values we're going
587  * to use to drive our operation, and so they are the authoritative
588  * values. Otherwise, we only store values into shared memory if shared
589  * memory is uninitialized. Our values are not canonical in such a case,
590  * but it's better to have something than nothing, to guide WAL retention.
591  */
592  LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
593  if (am_wal_summarizer || !WalSummarizerCtl->initialized)
594  {
596  WalSummarizerCtl->summarized_lsn = unsummarized_lsn;
597  WalSummarizerCtl->summarized_tli = unsummarized_tli;
598  WalSummarizerCtl->lsn_is_exact = should_make_exact;
599  WalSummarizerCtl->pending_lsn = unsummarized_lsn;
600  }
601  else
602  unsummarized_lsn = WalSummarizerCtl->summarized_lsn;
603 
604  /* Also return the to the caller as required. */
605  if (tli != NULL)
607  if (lsn_is_exact != NULL)
608  *lsn_is_exact = WalSummarizerCtl->lsn_is_exact;
609  LWLockRelease(WALSummarizerLock);
610 
611  return unsummarized_lsn;
612 }
613 
614 /*
615  * Attempt to set the WAL summarizer's latch.
616  *
617  * This might not work, because there's no guarantee that the WAL summarizer
618  * process was successfully started, and it also might have started but
619  * subsequently terminated. So, under normal circumstances, this will get the
620  * latch set, but there's no guarantee.
621  */
622 void
624 {
625  ProcNumber pgprocno;
626 
627  if (WalSummarizerCtl == NULL)
628  return;
629 
630  LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
632  LWLockRelease(WALSummarizerLock);
633 
634  if (pgprocno != INVALID_PROC_NUMBER)
635  SetLatch(&ProcGlobal->allProcs[pgprocno].procLatch);
636 }
637 
638 /*
639  * Wait until WAL summarization reaches the given LSN, but not longer than
640  * the given timeout.
641  *
642  * The return value is the first still-unsummarized LSN. If it's greater than
643  * or equal to the passed LSN, then that LSN was reached. If not, we timed out.
644  *
645  * Either way, *pending_lsn is set to the value taken from WalSummarizerCtl.
646  */
648 WaitForWalSummarization(XLogRecPtr lsn, long timeout, XLogRecPtr *pending_lsn)
649 {
652  XLogRecPtr summarized_lsn;
653 
655  Assert(timeout > 0);
656 
657  while (1)
658  {
660  long remaining_timeout;
661 
662  /*
663  * If the LSN summarized on disk has reached the target value, stop.
664  */
665  LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
666  summarized_lsn = WalSummarizerCtl->summarized_lsn;
667  *pending_lsn = WalSummarizerCtl->pending_lsn;
668  LWLockRelease(WALSummarizerLock);
669  if (summarized_lsn >= lsn)
670  break;
671 
672  /* Timeout reached? If yes, stop. */
674  remaining_timeout = TimestampDifferenceMilliseconds(now, deadline);
675  if (remaining_timeout <= 0)
676  break;
677 
678  /* Wait and see. */
680  remaining_timeout,
681  WAIT_EVENT_WAL_SUMMARY_READY);
682  }
683 
684  return summarized_lsn;
685 }
686 
687 /*
688  * On exit, update shared memory to make it clear that we're no longer
689  * running.
690  */
691 static void
693 {
694  LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
696  LWLockRelease(WALSummarizerLock);
697 }
698 
699 /*
700  * Get the latest LSN that is eligible to be summarized, and set *tli to the
701  * corresponding timeline.
702  */
703 static XLogRecPtr
705 {
706  if (!RecoveryInProgress())
707  {
708  /* Don't summarize WAL before it's flushed. */
709  return GetFlushRecPtr(tli);
710  }
711  else
712  {
713  XLogRecPtr flush_lsn;
714  TimeLineID flush_tli;
715  XLogRecPtr replay_lsn;
716  TimeLineID replay_tli;
717 
718  /*
719  * What we really want to know is how much WAL has been flushed to
720  * disk, but the only flush position available is the one provided by
721  * the walreceiver, which may not be running, because this could be
722  * crash recovery or recovery via restore_command. So use either the
723  * WAL receiver's flush position or the replay position, whichever is
724  * further ahead, on the theory that if the WAL has been replayed then
725  * it must also have been flushed to disk.
726  */
727  flush_lsn = GetWalRcvFlushRecPtr(NULL, &flush_tli);
728  replay_lsn = GetXLogReplayRecPtr(&replay_tli);
729  if (flush_lsn > replay_lsn)
730  {
731  *tli = flush_tli;
732  return flush_lsn;
733  }
734  else
735  {
736  *tli = replay_tli;
737  return replay_lsn;
738  }
739  }
740 }
741 
742 /*
743  * Interrupt handler for main loop of WAL summarizer process.
744  */
745 static void
747 {
750 
752  {
753  ConfigReloadPending = false;
755  }
756 
758  {
759  ereport(DEBUG1,
760  errmsg_internal("WAL summarizer shutting down"));
761  proc_exit(0);
762  }
763 
764  /* Perform logging of memory contexts of this process */
767 }
768 
769 /*
770  * Summarize a range of WAL records on a single timeline.
771  *
772  * 'tli' is the timeline to be summarized.
773  *
774  * 'start_lsn' is the point at which we should start summarizing. If this
775  * value comes from the end LSN of the previous record as returned by the
776  * xlogreader machinery, 'exact' should be true; otherwise, 'exact' should
777  * be false, and this function will search forward for the start of a valid
778  * WAL record.
779  *
780  * 'switch_lsn' is the point at which we should switch to a later timeline,
781  * if we're summarizing a historic timeline.
782  *
783  * 'maximum_lsn' identifies the point beyond which we can't count on being
784  * able to read any more WAL. It should be the switch point when reading a
785  * historic timeline, or the most-recently-measured end of WAL when reading
786  * the current timeline.
787  *
788  * The return value is the LSN at which the WAL summary actually ends. Most
789  * often, a summary file ends because we notice that a checkpoint has
790  * occurred and reach the redo pointer of that checkpoint, but sometimes
791  * we stop for other reasons, such as a timeline switch.
792  */
793 static XLogRecPtr
794 SummarizeWAL(TimeLineID tli, XLogRecPtr start_lsn, bool exact,
795  XLogRecPtr switch_lsn, XLogRecPtr maximum_lsn)
796 {
797  SummarizerReadLocalXLogPrivate *private_data;
799  XLogRecPtr summary_start_lsn;
800  XLogRecPtr summary_end_lsn = switch_lsn;
801  char temp_path[MAXPGPATH];
802  char final_path[MAXPGPATH];
803  WalSummaryIO io;
805 
806  /* Initialize private data for xlogreader. */
807  private_data = (SummarizerReadLocalXLogPrivate *)
809  private_data->tli = tli;
810  private_data->historic = !XLogRecPtrIsInvalid(switch_lsn);
811  private_data->read_upto = maximum_lsn;
812 
813  /* Create xlogreader. */
816  .segment_open = &wal_segment_open,
817  .segment_close = &wal_segment_close),
818  private_data);
819  if (xlogreader == NULL)
820  ereport(ERROR,
821  (errcode(ERRCODE_OUT_OF_MEMORY),
822  errmsg("out of memory"),
823  errdetail("Failed while allocating a WAL reading processor.")));
824 
825  /*
826  * When exact = false, we're starting from an arbitrary point in the WAL
827  * and must search forward for the start of the next record.
828  *
829  * When exact = true, start_lsn should be either the LSN where a record
830  * begins, or the LSN of a page where the page header is immediately
831  * followed by the start of a new record. XLogBeginRead should tolerate
832  * either case.
833  *
834  * We need to allow for both cases because the behavior of xlogreader
835  * varies. When a record spans two or more xlog pages, the ending LSN
836  * reported by xlogreader will be the starting LSN of the following
837  * record, but when an xlog page boundary falls between two records, the
838  * end LSN for the first will be reported as the first byte of the
839  * following page. We can't know until we read that page how large the
840  * header will be, but we'll have to skip over it to find the next record.
841  */
842  if (exact)
843  {
844  /*
845  * Even if start_lsn is the beginning of a page rather than the
846  * beginning of the first record on that page, we should still use it
847  * as the start LSN for the summary file. That's because we detect
848  * missing summary files by looking for cases where the end LSN of one
849  * file is less than the start LSN of the next file. When only a page
850  * header is skipped, nothing has been missed.
851  */
852  XLogBeginRead(xlogreader, start_lsn);
853  summary_start_lsn = start_lsn;
854  }
855  else
856  {
857  summary_start_lsn = XLogFindNextRecord(xlogreader, start_lsn);
858  if (XLogRecPtrIsInvalid(summary_start_lsn))
859  {
860  /*
861  * If we hit end-of-WAL while trying to find the next valid
862  * record, we must be on a historic timeline that has no valid
863  * records that begin after start_lsn and before end of WAL.
864  */
865  if (private_data->end_of_wal)
866  {
867  ereport(DEBUG1,
868  errmsg_internal("could not read WAL from timeline %u at %X/%X: end of WAL at %X/%X",
869  tli,
870  LSN_FORMAT_ARGS(start_lsn),
871  LSN_FORMAT_ARGS(private_data->read_upto)));
872 
873  /*
874  * The timeline ends at or after start_lsn, without containing
875  * any records. Thus, we must make sure the main loop does not
876  * iterate. If start_lsn is the end of the timeline, then we
877  * won't actually emit an empty summary file, but otherwise,
878  * we must, to capture the fact that the LSN range in question
879  * contains no interesting WAL records.
880  */
881  summary_start_lsn = start_lsn;
882  summary_end_lsn = private_data->read_upto;
883  switch_lsn = xlogreader->EndRecPtr;
884  }
885  else
886  ereport(ERROR,
887  (errmsg("could not find a valid record after %X/%X",
888  LSN_FORMAT_ARGS(start_lsn))));
889  }
890 
891  /* We shouldn't go backward. */
892  Assert(summary_start_lsn >= start_lsn);
893  }
894 
895  /*
896  * Main loop: read xlog records one by one.
897  */
898  while (1)
899  {
900  int block_id;
901  char *errormsg;
902  XLogRecord *record;
903  bool stop_requested = false;
904 
906 
907  /* We shouldn't go backward. */
908  Assert(summary_start_lsn <= xlogreader->EndRecPtr);
909 
910  /* Now read the next record. */
911  record = XLogReadRecord(xlogreader, &errormsg);
912  if (record == NULL)
913  {
914  if (private_data->end_of_wal)
915  {
916  /*
917  * This timeline must be historic and must end before we were
918  * able to read a complete record.
919  */
920  ereport(DEBUG1,
921  errmsg_internal("could not read WAL from timeline %u at %X/%X: end of WAL at %X/%X",
922  tli,
924  LSN_FORMAT_ARGS(private_data->read_upto)));
925  /* Summary ends at end of WAL. */
926  summary_end_lsn = private_data->read_upto;
927  break;
928  }
929  if (errormsg)
930  ereport(ERROR,
932  errmsg("could not read WAL from timeline %u at %X/%X: %s",
934  errormsg)));
935  else
936  ereport(ERROR,
938  errmsg("could not read WAL from timeline %u at %X/%X",
940  }
941 
942  /* We shouldn't go backward. */
943  Assert(summary_start_lsn <= xlogreader->EndRecPtr);
944 
945  if (!XLogRecPtrIsInvalid(switch_lsn) &&
946  xlogreader->ReadRecPtr >= switch_lsn)
947  {
948  /*
949  * Whoops! We've read a record that *starts* after the switch LSN,
950  * contrary to our goal of reading only until we hit the first
951  * record that ends at or after the switch LSN. Pretend we didn't
952  * read it after all by bailing out of this loop right here,
953  * before we do anything with this record.
954  *
955  * This can happen because the last record before the switch LSN
956  * might be continued across multiple pages, and then we might
957  * come to a page with XLP_FIRST_IS_OVERWRITE_CONTRECORD set. In
958  * that case, the record that was continued across multiple pages
959  * is incomplete and will be disregarded, and the read will
960  * restart from the beginning of the page that is flagged
961  * XLP_FIRST_IS_OVERWRITE_CONTRECORD.
962  *
963  * If this case occurs, we can fairly say that the current summary
964  * file ends at the switch LSN exactly. The first record on the
965  * page marked XLP_FIRST_IS_OVERWRITE_CONTRECORD will be
966  * discovered when generating the next summary file.
967  */
968  summary_end_lsn = switch_lsn;
969  break;
970  }
971 
972  /* Special handling for particular types of WAL records. */
973  switch (XLogRecGetRmid(xlogreader))
974  {
975  case RM_DBASE_ID:
977  break;
978  case RM_SMGR_ID:
980  break;
981  case RM_XACT_ID:
983  break;
984  case RM_XLOG_ID:
985  stop_requested = SummarizeXlogRecord(xlogreader);
986  break;
987  default:
988  break;
989  }
990 
991  /*
992  * If we've been told that it's time to end this WAL summary file, do
993  * so. As an exception, if there's nothing included in this WAL
994  * summary file yet, then stopping doesn't make any sense, and we
995  * should wait until the next stop point instead.
996  */
997  if (stop_requested && xlogreader->ReadRecPtr > summary_start_lsn)
998  {
999  summary_end_lsn = xlogreader->ReadRecPtr;
1000  break;
1001  }
1002 
1003  /* Feed block references from xlog record to block reference table. */
1004  for (block_id = 0; block_id <= XLogRecMaxBlockId(xlogreader);
1005  block_id++)
1006  {
1007  RelFileLocator rlocator;
1008  ForkNumber forknum;
1009  BlockNumber blocknum;
1010 
1011  if (!XLogRecGetBlockTagExtended(xlogreader, block_id, &rlocator,
1012  &forknum, &blocknum, NULL))
1013  continue;
1014 
1015  /*
1016  * As we do elsewhere, ignore the FSM fork, because it's not fully
1017  * WAL-logged.
1018  */
1019  if (forknum != FSM_FORKNUM)
1020  BlockRefTableMarkBlockModified(brtab, &rlocator, forknum,
1021  blocknum);
1022  }
1023 
1024  /* Update our notion of where this summary file ends. */
1025  summary_end_lsn = xlogreader->EndRecPtr;
1026 
1027  /* Also update shared memory. */
1028  LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
1029  Assert(summary_end_lsn >= WalSummarizerCtl->pending_lsn);
1030  Assert(summary_end_lsn >= WalSummarizerCtl->summarized_lsn);
1031  WalSummarizerCtl->pending_lsn = summary_end_lsn;
1032  LWLockRelease(WALSummarizerLock);
1033 
1034  /*
1035  * If we have a switch LSN and have reached it, stop before reading
1036  * the next record.
1037  */
1038  if (!XLogRecPtrIsInvalid(switch_lsn) &&
1039  xlogreader->EndRecPtr >= switch_lsn)
1040  break;
1041  }
1042 
1043  /* Destroy xlogreader. */
1046 
1047  /*
1048  * If a timeline switch occurs, we may fail to make any progress at all
1049  * before exiting the loop above. If that happens, we don't write a WAL
1050  * summary file at all.
1051  */
1052  if (summary_end_lsn > summary_start_lsn)
1053  {
1054  /* Generate temporary and final path name. */
1055  snprintf(temp_path, MAXPGPATH,
1056  XLOGDIR "/summaries/temp.summary");
1057  snprintf(final_path, MAXPGPATH,
1058  XLOGDIR "/summaries/%08X%08X%08X%08X%08X.summary",
1059  tli,
1060  LSN_FORMAT_ARGS(summary_start_lsn),
1061  LSN_FORMAT_ARGS(summary_end_lsn));
1062 
1063  /* Open the temporary file for writing. */
1064  io.filepos = 0;
1065  io.file = PathNameOpenFile(temp_path, O_WRONLY | O_CREAT | O_TRUNC);
1066  if (io.file < 0)
1067  ereport(ERROR,
1069  errmsg("could not create file \"%s\": %m", temp_path)));
1070 
1071  /* Write the data. */
1072  WriteBlockRefTable(brtab, WriteWalSummary, &io);
1073 
1074  /* Close temporary file and shut down xlogreader. */
1075  FileClose(io.file);
1076 
1077  /* Tell the user what we did. */
1078  ereport(DEBUG1,
1079  errmsg("summarized WAL on TLI %u from %X/%X to %X/%X",
1080  tli,
1081  LSN_FORMAT_ARGS(summary_start_lsn),
1082  LSN_FORMAT_ARGS(summary_end_lsn)));
1083 
1084  /* Durably rename the new summary into place. */
1085  durable_rename(temp_path, final_path, ERROR);
1086  }
1087 
1088  return summary_end_lsn;
1089 }
1090 
1091 /*
1092  * Special handling for WAL records with RM_DBASE_ID.
1093  */
1094 static void
1096 {
1098 
1099  /*
1100  * We use relfilenode zero for a given database OID and tablespace OID to
1101  * indicate that all relations with that pair of IDs have been recreated
1102  * if they exist at all. Effectively, we're setting a limit block of 0 for
1103  * all such relfilenodes.
1104  *
1105  * Technically, this special handling is only needed in the case of
1106  * XLOG_DBASE_CREATE_FILE_COPY, because that can create a whole bunch of
1107  * relation files in a directory without logging anything specific to each
1108  * one. If we didn't mark the whole DB OID/TS OID combination in some way,
1109  * then a tablespace that was dropped after the reference backup and
1110  * recreated using the FILE_COPY method prior to the incremental backup
1111  * would look just like one that was never touched at all, which would be
1112  * catastrophic.
1113  *
1114  * But it seems best to adopt this treatment for all records that drop or
1115  * create a DB OID/TS OID combination. That's similar to how we treat the
1116  * limit block for individual relations, and it's an extra layer of safety
1117  * here. We can never lose data by marking more stuff as needing to be
1118  * backed up in full.
1119  */
1120  if (info == XLOG_DBASE_CREATE_FILE_COPY)
1121  {
1123  RelFileLocator rlocator;
1124 
1125  xlrec =
1127  rlocator.spcOid = xlrec->tablespace_id;
1128  rlocator.dbOid = xlrec->db_id;
1129  rlocator.relNumber = 0;
1130  BlockRefTableSetLimitBlock(brtab, &rlocator, MAIN_FORKNUM, 0);
1131  }
1132  else if (info == XLOG_DBASE_CREATE_WAL_LOG)
1133  {
1135  RelFileLocator rlocator;
1136 
1138  rlocator.spcOid = xlrec->tablespace_id;
1139  rlocator.dbOid = xlrec->db_id;
1140  rlocator.relNumber = 0;
1141  BlockRefTableSetLimitBlock(brtab, &rlocator, MAIN_FORKNUM, 0);
1142  }
1143  else if (info == XLOG_DBASE_DROP)
1144  {
1145  xl_dbase_drop_rec *xlrec;
1146  RelFileLocator rlocator;
1147  int i;
1148 
1150  rlocator.dbOid = xlrec->db_id;
1151  rlocator.relNumber = 0;
1152  for (i = 0; i < xlrec->ntablespaces; ++i)
1153  {
1154  rlocator.spcOid = xlrec->tablespace_ids[i];
1155  BlockRefTableSetLimitBlock(brtab, &rlocator, MAIN_FORKNUM, 0);
1156  }
1157  }
1158 }
1159 
1160 /*
1161  * Special handling for WAL records with RM_SMGR_ID.
1162  */
1163 static void
1165 {
1167 
1168  if (info == XLOG_SMGR_CREATE)
1169  {
1170  xl_smgr_create *xlrec;
1171 
1172  /*
1173  * If a new relation fork is created on disk, there is no point
1174  * tracking anything about which blocks have been modified, because
1175  * the whole thing will be new. Hence, set the limit block for this
1176  * fork to 0.
1177  *
1178  * Ignore the FSM fork, which is not fully WAL-logged.
1179  */
1181 
1182  if (xlrec->forkNum != FSM_FORKNUM)
1183  BlockRefTableSetLimitBlock(brtab, &xlrec->rlocator,
1184  xlrec->forkNum, 0);
1185  }
1186  else if (info == XLOG_SMGR_TRUNCATE)
1187  {
1188  xl_smgr_truncate *xlrec;
1189 
1191 
1192  /*
1193  * If a relation fork is truncated on disk, there is no point in
1194  * tracking anything about block modifications beyond the truncation
1195  * point.
1196  *
1197  * We ignore SMGR_TRUNCATE_FSM here because the FSM isn't fully
1198  * WAL-logged and thus we can't track modified blocks for it anyway.
1199  */
1200  if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
1201  BlockRefTableSetLimitBlock(brtab, &xlrec->rlocator,
1202  MAIN_FORKNUM, xlrec->blkno);
1203  if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0)
1204  BlockRefTableSetLimitBlock(brtab, &xlrec->rlocator,
1205  VISIBILITYMAP_FORKNUM, xlrec->blkno);
1206  }
1207 }
1208 
1209 /*
1210  * Special handling for WAL records with RM_XACT_ID.
1211  */
1212 static void
1214 {
1216  uint8 xact_info = info & XLOG_XACT_OPMASK;
1217 
1218  if (xact_info == XLOG_XACT_COMMIT ||
1219  xact_info == XLOG_XACT_COMMIT_PREPARED)
1220  {
1222  xl_xact_parsed_commit parsed;
1223  int i;
1224 
1225  /*
1226  * Don't track modified blocks for any relations that were removed on
1227  * commit.
1228  */
1229  ParseCommitRecord(XLogRecGetInfo(xlogreader), xlrec, &parsed);
1230  for (i = 0; i < parsed.nrels; ++i)
1231  {
1232  ForkNumber forknum;
1233 
1234  for (forknum = 0; forknum <= MAX_FORKNUM; ++forknum)
1235  if (forknum != FSM_FORKNUM)
1236  BlockRefTableSetLimitBlock(brtab, &parsed.xlocators[i],
1237  forknum, 0);
1238  }
1239  }
1240  else if (xact_info == XLOG_XACT_ABORT ||
1241  xact_info == XLOG_XACT_ABORT_PREPARED)
1242  {
1244  xl_xact_parsed_abort parsed;
1245  int i;
1246 
1247  /*
1248  * Don't track modified blocks for any relations that were removed on
1249  * abort.
1250  */
1251  ParseAbortRecord(XLogRecGetInfo(xlogreader), xlrec, &parsed);
1252  for (i = 0; i < parsed.nrels; ++i)
1253  {
1254  ForkNumber forknum;
1255 
1256  for (forknum = 0; forknum <= MAX_FORKNUM; ++forknum)
1257  if (forknum != FSM_FORKNUM)
1258  BlockRefTableSetLimitBlock(brtab, &parsed.xlocators[i],
1259  forknum, 0);
1260  }
1261  }
1262 }
1263 
1264 /*
1265  * Special handling for WAL records with RM_XLOG_ID.
1266  */
1267 static bool
1269 {
1271 
1272  if (info == XLOG_CHECKPOINT_REDO || info == XLOG_CHECKPOINT_SHUTDOWN)
1273  {
1274  /*
1275  * This is an LSN at which redo might begin, so we'd like
1276  * summarization to stop just before this WAL record.
1277  */
1278  return true;
1279  }
1280 
1281  return false;
1282 }
1283 
1284 /*
1285  * Similar to read_local_xlog_page, but limited to read from one particular
1286  * timeline. If the end of WAL is reached, it will wait for more if reading
1287  * from the current timeline, or give up if reading from a historic timeline.
1288  * In the latter case, it will also set private_data->end_of_wal = true.
1289  *
1290  * Caller must set private_data->tli to the TLI of interest,
1291  * private_data->read_upto to the lowest LSN that is not known to be safe
1292  * to read on that timeline, and private_data->historic to true if and only
1293  * if the timeline is not the current timeline. This function will update
1294  * private_data->read_upto and private_data->historic if more WAL appears
1295  * on the current timeline or if the current timeline becomes historic.
1296  */
1297 static int
1299  XLogRecPtr targetPagePtr, int reqLen,
1300  XLogRecPtr targetRecPtr, char *cur_page)
1301 {
1302  int count;
1303  WALReadError errinfo;
1304  SummarizerReadLocalXLogPrivate *private_data;
1305 
1307 
1308  private_data = (SummarizerReadLocalXLogPrivate *)
1309  state->private_data;
1310 
1311  while (1)
1312  {
1313  if (targetPagePtr + XLOG_BLCKSZ <= private_data->read_upto)
1314  {
1315  /*
1316  * more than one block available; read only that block, have
1317  * caller come back if they need more.
1318  */
1319  count = XLOG_BLCKSZ;
1320  break;
1321  }
1322  else if (targetPagePtr + reqLen > private_data->read_upto)
1323  {
1324  /* We don't seem to have enough data. */
1325  if (private_data->historic)
1326  {
1327  /*
1328  * This is a historic timeline, so there will never be any
1329  * more data than we have currently.
1330  */
1331  private_data->end_of_wal = true;
1332  return -1;
1333  }
1334  else
1335  {
1336  XLogRecPtr latest_lsn;
1337  TimeLineID latest_tli;
1338 
1339  /*
1340  * This is - or at least was up until very recently - the
1341  * current timeline, so more data might show up. Delay here
1342  * so we don't tight-loop.
1343  */
1346 
1347  /* Recheck end-of-WAL. */
1348  latest_lsn = GetLatestLSN(&latest_tli);
1349  if (private_data->tli == latest_tli)
1350  {
1351  /* Still the current timeline, update max LSN. */
1352  Assert(latest_lsn >= private_data->read_upto);
1353  private_data->read_upto = latest_lsn;
1354  }
1355  else
1356  {
1357  List *tles = readTimeLineHistory(latest_tli);
1358  XLogRecPtr switchpoint;
1359 
1360  /*
1361  * The timeline we're scanning is no longer the latest
1362  * one. Figure out when it ended.
1363  */
1364  private_data->historic = true;
1365  switchpoint = tliSwitchPoint(private_data->tli, tles,
1366  NULL);
1367 
1368  /*
1369  * Allow reads up to exactly the switch point.
1370  *
1371  * It's possible that this will cause read_upto to move
1372  * backwards, because walreceiver might have read a
1373  * partial record and flushed it to disk, and we'd view
1374  * that data as safe to read. However, the
1375  * XLOG_END_OF_RECOVERY record will be written at the end
1376  * of the last complete WAL record, not at the end of the
1377  * WAL that we've flushed to disk.
1378  *
1379  * So switchpoint < private->read_upto is possible here,
1380  * but switchpoint < state->EndRecPtr should not be.
1381  */
1382  Assert(switchpoint >= state->EndRecPtr);
1383  private_data->read_upto = switchpoint;
1384 
1385  /* Debugging output. */
1386  ereport(DEBUG1,
1387  errmsg("timeline %u became historic, can read up to %X/%X",
1388  private_data->tli, LSN_FORMAT_ARGS(private_data->read_upto)));
1389  }
1390 
1391  /* Go around and try again. */
1392  }
1393  }
1394  else
1395  {
1396  /* enough bytes available to satisfy the request */
1397  count = private_data->read_upto - targetPagePtr;
1398  break;
1399  }
1400  }
1401 
1402  if (!WALRead(state, cur_page, targetPagePtr, count,
1403  private_data->tli, &errinfo))
1404  WALReadRaiseError(&errinfo);
1405 
1406  /* Track that we read a page, for sleep time calculation. */
1408 
1409  /* number of valid bytes in the buffer */
1410  return count;
1411 }
1412 
1413 /*
1414  * Sleep for long enough that we believe it's likely that more WAL will
1415  * be available afterwards.
1416  */
1417 static void
1419 {
1420  if (pages_read_since_last_sleep == 0)
1421  {
1422  /*
1423  * No pages were read since the last sleep, so double the sleep time,
1424  * but not beyond the maximum allowable value.
1425  */
1427  }
1428  else if (pages_read_since_last_sleep > 1)
1429  {
1430  /*
1431  * Multiple pages were read since the last sleep, so reduce the sleep
1432  * time.
1433  *
1434  * A large burst of activity should be able to quickly reduce the
1435  * sleep time to the minimum, but we don't want a handful of extra WAL
1436  * records to provoke a strong reaction. We choose to reduce the sleep
1437  * time by 1 quantum for each page read beyond the first, which is a
1438  * fairly arbitrary way of trying to be reactive without overreacting.
1439  */
1441  sleep_quanta = 1;
1442  else
1444  }
1445 
1446  /* OK, now sleep. */
1447  (void) WaitLatch(MyLatch,
1450  WAIT_EVENT_WAL_SUMMARIZER_WAL);
1452 
1453  /* Reset count of pages read. */
1455 }
1456 
1457 /*
1458  * Remove WAL summaries whose mtimes are older than wal_summary_keep_time.
1459  */
1460 static void
1462 {
1463  XLogRecPtr redo_pointer = GetRedoRecPtr();
1464  List *wslist;
1465  time_t cutoff_time;
1466 
1467  /* If WAL summary removal is disabled, don't do anything. */
1468  if (wal_summary_keep_time == 0)
1469  return;
1470 
1471  /*
1472  * If the redo pointer has not advanced, don't do anything.
1473  *
1474  * This has the effect that we only try to remove old WAL summary files
1475  * once per checkpoint cycle.
1476  */
1477  if (redo_pointer == redo_pointer_at_last_summary_removal)
1478  return;
1479  redo_pointer_at_last_summary_removal = redo_pointer;
1480 
1481  /*
1482  * Files should only be removed if the last modification time precedes the
1483  * cutoff time we compute here.
1484  */
1485  cutoff_time = time(NULL) - wal_summary_keep_time * SECS_PER_MINUTE;
1486 
1487  /* Get all the summaries that currently exist. */
1489 
1490  /* Loop until all summaries have been considered for removal. */
1491  while (wslist != NIL)
1492  {
1493  ListCell *lc;
1494  XLogSegNo oldest_segno;
1495  XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
1496  TimeLineID selected_tli;
1497 
1499 
1500  /*
1501  * Pick a timeline for which some summary files still exist on disk,
1502  * and find the oldest LSN that still exists on disk for that
1503  * timeline.
1504  */
1505  selected_tli = ((WalSummaryFile *) linitial(wslist))->tli;
1506  oldest_segno = XLogGetOldestSegno(selected_tli);
1507  if (oldest_segno != 0)
1508  XLogSegNoOffsetToRecPtr(oldest_segno, 0, wal_segment_size,
1509  oldest_lsn);
1510 
1511 
1512  /* Consider each WAL file on the selected timeline in turn. */
1513  foreach(lc, wslist)
1514  {
1515  WalSummaryFile *ws = lfirst(lc);
1516 
1518 
1519  /* If it's not on this timeline, it's not time to consider it. */
1520  if (selected_tli != ws->tli)
1521  continue;
1522 
1523  /*
1524  * If the WAL doesn't exist any more, we can remove it if the file
1525  * modification time is old enough.
1526  */
1527  if (XLogRecPtrIsInvalid(oldest_lsn) || ws->end_lsn <= oldest_lsn)
1528  RemoveWalSummaryIfOlderThan(ws, cutoff_time);
1529 
1530  /*
1531  * Whether we removed the file or not, we need not consider it
1532  * again.
1533  */
1534  wslist = foreach_delete_current(wslist, lc);
1535  pfree(ws);
1536  }
1537  }
1538 }
void AuxiliaryProcessMainCommon(void)
Definition: auxprocess.c:44
XLogRecPtr tliSwitchPoint(TimeLineID tli, List *history, TimeLineID *nextTLI)
Definition: timeline.c:572
List * readTimeLineHistory(TimeLineID targetTLI)
Definition: timeline.c:76
sigset_t UnBlockSig
Definition: pqsignal.c:22
long TimestampDifferenceMilliseconds(TimestampTz start_time, TimestampTz stop_time)
Definition: timestamp.c:1767
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1655
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1619
void BlockRefTableMarkBlockModified(BlockRefTable *brtab, const RelFileLocator *rlocator, ForkNumber forknum, BlockNumber blknum)
Definition: blkreftable.c:297
void BlockRefTableSetLimitBlock(BlockRefTable *brtab, const RelFileLocator *rlocator, ForkNumber forknum, BlockNumber limit_block)
Definition: blkreftable.c:262
void WriteBlockRefTable(BlockRefTable *brtab, io_callback_fn write_callback, void *write_callback_arg)
Definition: blkreftable.c:474
void(*) BlockRefTable CreateEmptyBlockRefTable)(void)
uint32 BlockNumber
Definition: block.h:31
#define Min(x, y)
Definition: c.h:1004
#define Assert(condition)
Definition: c.h:858
unsigned char uint8
Definition: c.h:504
size_t Size
Definition: c.h:605
bool ConditionVariableCancelSleep(void)
bool ConditionVariableTimedSleep(ConditionVariable *cv, long timeout, uint32 wait_event_info)
void ConditionVariableBroadcast(ConditionVariable *cv)
void ConditionVariableInit(ConditionVariable *cv)
int64 TimestampTz
Definition: timestamp.h:39
#define MINS_PER_HOUR
Definition: timestamp.h:129
#define SECS_PER_MINUTE
Definition: timestamp.h:128
#define HOURS_PER_DAY
Definition: timestamp.h:118
#define XLOG_DBASE_CREATE_WAL_LOG
#define XLOG_DBASE_DROP
#define XLOG_DBASE_CREATE_FILE_COPY
void AtEOXact_HashTables(bool isCommit)
Definition: dynahash.c:1869
int errmsg_internal(const char *fmt,...)
Definition: elog.c:1155
void EmitErrorReport(void)
Definition: elog.c:1668
int errcode_for_file_access(void)
Definition: elog.c:878
int errdetail(const char *fmt,...)
Definition: elog.c:1201
ErrorContextCallback * error_context_stack
Definition: elog.c:94
void FlushErrorState(void)
Definition: elog.c:1848
int errcode(int sqlerrcode)
Definition: elog.c:855
int errmsg(const char *fmt,...)
Definition: elog.c:1068
sigjmp_buf * PG_exception_stack
Definition: elog.c:96
#define DEBUG1
Definition: elog.h:30
#define ERROR
Definition: elog.h:39
#define ereport(elevel,...)
Definition: elog.h:149
int durable_rename(const char *oldfile, const char *newfile, int elevel)
Definition: fd.c:782
void AtEOXact_Files(bool isCommit)
Definition: fd.c:3165
void FileClose(File file)
Definition: fd.c:1978
File PathNameOpenFile(const char *fileName, int fileFlags)
Definition: fd.c:1575
volatile sig_atomic_t LogMemoryContextPending
Definition: globals.c:40
volatile sig_atomic_t ProcSignalBarrierPending
Definition: globals.c:39
ProcNumber MyProcNumber
Definition: globals.c:88
struct Latch * MyLatch
Definition: globals.c:61
@ PGC_SIGHUP
Definition: guc.h:71
void ProcessConfigFile(GucContext context)
void SignalHandlerForShutdownRequest(SIGNAL_ARGS)
Definition: interrupt.c:105
volatile sig_atomic_t ShutdownRequestPending
Definition: interrupt.c:28
volatile sig_atomic_t ConfigReloadPending
Definition: interrupt.c:27
void SignalHandlerForConfigReload(SIGNAL_ARGS)
Definition: interrupt.c:61
void on_shmem_exit(pg_on_exit_callback function, Datum arg)
Definition: ipc.c:365
void proc_exit(int code)
Definition: ipc.c:104
int i
Definition: isn.c:73
void SetLatch(Latch *latch)
Definition: latch.c:632
void ResetLatch(Latch *latch)
Definition: latch.c:724
int WaitLatch(Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:517
#define WL_TIMEOUT
Definition: latch.h:130
#define WL_EXIT_ON_PM_DEATH
Definition: latch.h:132
#define WL_LATCH_SET
Definition: latch.h:127
bool LWLockAcquire(LWLock *lock, LWLockMode mode)
Definition: lwlock.c:1168
void LWLockRelease(LWLock *lock)
Definition: lwlock.c:1781
void LWLockReleaseAll(void)
Definition: lwlock.c:1876
@ LW_SHARED
Definition: lwlock.h:115
@ LW_EXCLUSIVE
Definition: lwlock.h:114
void MemoryContextReset(MemoryContext context)
Definition: mcxt.c:383
void pfree(void *pointer)
Definition: mcxt.c:1521
MemoryContext TopMemoryContext
Definition: mcxt.c:149
void * palloc0(Size size)
Definition: mcxt.c:1347
void ProcessLogMemoryContextInterrupt(void)
Definition: mcxt.c:1289
#define AllocSetContextCreate
Definition: memutils.h:129
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:160
#define RESUME_INTERRUPTS()
Definition: miscadmin.h:135
#define AmWalSummarizerProcess()
Definition: miscadmin.h:384
#define HOLD_INTERRUPTS()
Definition: miscadmin.h:133
@ B_WAL_SUMMARIZER
Definition: miscadmin.h:360
BackendType MyBackendType
Definition: miscinit.c:63
void * arg
#define MAXPGPATH
#define XLOG_CHECKPOINT_REDO
Definition: pg_control.h:81
#define XLOG_CHECKPOINT_SHUTDOWN
Definition: pg_control.h:67
while(p+4<=pend)
static time_t start_time
Definition: pg_ctl.c:94
#define lfirst(lc)
Definition: pg_list.h:172
static int list_length(const List *l)
Definition: pg_list.h:152
#define NIL
Definition: pg_list.h:68
#define foreach_delete_current(lst, var_or_cell)
Definition: pg_list.h:391
#define linitial(l)
Definition: pg_list.h:178
static void * list_nth(const List *list, int n)
Definition: pg_list.h:299
pqsigfunc pqsignal(int signo, pqsigfunc func)
#define snprintf
Definition: port.h:238
uintptr_t Datum
Definition: postgres.h:64
#define GetPGProcByNumber(n)
Definition: proc.h:427
#define INVALID_PROC_NUMBER
Definition: procnumber.h:26
int ProcNumber
Definition: procnumber.h:24
void ProcessProcSignalBarrier(void)
Definition: procsignal.c:464
void procsignal_sigusr1_handler(SIGNAL_ARGS)
Definition: procsignal.c:635
tree context
Definition: radixtree.h:1835
MemoryContextSwitchTo(old_ctx)
ForkNumber
Definition: relpath.h:48
@ FSM_FORKNUM
Definition: relpath.h:51
@ VISIBILITYMAP_FORKNUM
Definition: relpath.h:52
@ MAIN_FORKNUM
Definition: relpath.h:50
#define MAX_FORKNUM
Definition: relpath.h:62
void ReleaseAuxProcessResources(bool isCommit)
Definition: resowner.c:1002
void * ShmemInitStruct(const char *name, Size size, bool *foundPtr)
Definition: shmem.c:387
PROC_HDR * ProcGlobal
Definition: proc.c:78
#define SMGR_TRUNCATE_VM
Definition: storage_xlog.h:41
#define XLOG_SMGR_CREATE
Definition: storage_xlog.h:30
#define XLOG_SMGR_TRUNCATE
Definition: storage_xlog.h:31
#define SMGR_TRUNCATE_HEAP
Definition: storage_xlog.h:40
Definition: pg_list.h:54
Latch procLatch
Definition: proc.h:164
PGPROC * allProcs
Definition: proc.h:379
RelFileNumber relNumber
TimeLineID tli
Definition: timeline.h:27
XLogRecPtr summarized_lsn
Definition: walsummarizer.c:84
TimeLineID summarized_tli
Definition: walsummarizer.c:83
ConditionVariable summary_file_cv
Definition: walsummarizer.c:92
ProcNumber summarizer_pgprocno
Definition: walsummarizer.c:86
XLogRecPtr pending_lsn
Definition: walsummarizer.c:87
XLogRecPtr end_lsn
Definition: walsummary.h:30
TimeLineID tli
Definition: walsummary.h:31
off_t filepos
Definition: walsummary.h:24
XLogRecPtr EndRecPtr
Definition: xlogreader.h:207
XLogRecPtr ReadRecPtr
Definition: xlogreader.h:206
void * private_data
Definition: xlogreader.h:196
Definition: regguts.h:323
Oid tablespace_ids[FLEXIBLE_ARRAY_MEMBER]
ForkNumber forkNum
Definition: storage_xlog.h:36
RelFileLocator rlocator
Definition: storage_xlog.h:35
RelFileLocator rlocator
Definition: storage_xlog.h:49
BlockNumber blkno
Definition: storage_xlog.h:48
RelFileLocator * xlocators
Definition: xact.h:416
RelFileLocator * xlocators
Definition: xact.h:383
#define TimestampTzPlusMilliseconds(tz, ms)
Definition: timestamp.h:85
static void pgstat_report_wait_end(void)
Definition: wait_event.h:101
XLogRecPtr GetWalRcvFlushRecPtr(XLogRecPtr *latestChunkStart, TimeLineID *receiveTLI)
void SetWalSummarizerLatch(void)
static XLogRecPtr redo_pointer_at_last_summary_removal
#define MAX_SLEEP_QUANTA
static long pages_read_since_last_sleep
XLogRecPtr WaitForWalSummarization(XLogRecPtr lsn, long timeout, XLogRecPtr *pending_lsn)
void WalSummarizerMain(char *startup_data, size_t startup_data_len)
static bool SummarizeXlogRecord(XLogReaderState *xlogreader)
Size WalSummarizerShmemSize(void)
static XLogRecPtr GetLatestLSN(TimeLineID *tli)
static XLogRecPtr SummarizeWAL(TimeLineID tli, XLogRecPtr start_lsn, bool exact, XLogRecPtr switch_lsn, XLogRecPtr maximum_lsn)
static WalSummarizerData * WalSummarizerCtl
static void SummarizeXactRecord(XLogReaderState *xlogreader, BlockRefTable *brtab)
bool summarize_wal
static void SummarizeDbaseRecord(XLogReaderState *xlogreader, BlockRefTable *brtab)
static void HandleWalSummarizerInterrupts(void)
#define MS_PER_SLEEP_QUANTUM
void GetWalSummarizerState(TimeLineID *summarized_tli, XLogRecPtr *summarized_lsn, XLogRecPtr *pending_lsn, int *summarizer_pid)
static long sleep_quanta
int wal_summary_keep_time
static int summarizer_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
static void WalSummarizerShutdown(int code, Datum arg)
static void SummarizeSmgrRecord(XLogReaderState *xlogreader, BlockRefTable *brtab)
static void MaybeRemoveOldWalSummaries(void)
XLogRecPtr GetOldestUnsummarizedLSN(TimeLineID *tli, bool *lsn_is_exact)
void WalSummarizerShmemInit(void)
static void summarizer_wait_for_wal(void)
void RemoveWalSummaryIfOlderThan(WalSummaryFile *ws, time_t cutoff_time)
Definition: walsummary.c:230
List * GetWalSummaries(TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
Definition: walsummary.c:43
int WriteWalSummary(void *wal_summary_io, void *data, int length)
Definition: walsummary.c:294
#define SIGCHLD
Definition: win32_port.h:178
#define SIGHUP
Definition: win32_port.h:168
#define SIG_DFL
Definition: win32_port.h:163
#define SIGPIPE
Definition: win32_port.h:173
#define SIGUSR1
Definition: win32_port.h:180
#define SIGALRM
Definition: win32_port.h:174
#define SIGUSR2
Definition: win32_port.h:181
#define SIG_IGN
Definition: win32_port.h:165
#define XLOG_XACT_COMMIT_PREPARED
Definition: xact.h:172
#define XLOG_XACT_COMMIT
Definition: xact.h:169
#define XLOG_XACT_OPMASK
Definition: xact.h:179
#define XLOG_XACT_ABORT
Definition: xact.h:171
#define XLOG_XACT_ABORT_PREPARED
Definition: xact.h:173
void ParseCommitRecord(uint8 info, xl_xact_commit *xlrec, xl_xact_parsed_commit *parsed)
Definition: xactdesc.c:35
void ParseAbortRecord(uint8 info, xl_xact_abort *xlrec, xl_xact_parsed_abort *parsed)
Definition: xactdesc.c:141
bool RecoveryInProgress(void)
Definition: xlog.c:6291
XLogRecPtr GetRedoRecPtr(void)
Definition: xlog.c:6394
int wal_segment_size
Definition: xlog.c:143
XLogRecPtr GetFlushRecPtr(TimeLineID *insertTLI)
Definition: xlog.c:6456
XLogSegNo XLogGetOldestSegno(TimeLineID tli)
Definition: xlog.c:3763
#define XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest)
#define XLOGDIR
#define LSN_FORMAT_ARGS(lsn)
Definition: xlogdefs.h:43
#define XLogRecPtrIsInvalid(r)
Definition: xlogdefs.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28
uint32 TimeLineID
Definition: xlogdefs.h:59
uint64 XLogSegNo
Definition: xlogdefs.h:48
bool XLogRecGetBlockTagExtended(XLogReaderState *record, uint8 block_id, RelFileLocator *rlocator, ForkNumber *forknum, BlockNumber *blknum, Buffer *prefetch_buffer)
Definition: xlogreader.c:1997
XLogRecord * XLogReadRecord(XLogReaderState *state, char **errormsg)
Definition: xlogreader.c:389
bool WALRead(XLogReaderState *state, char *buf, XLogRecPtr startptr, Size count, TimeLineID tli, WALReadError *errinfo)
Definition: xlogreader.c:1503
void XLogReaderFree(XLogReaderState *state)
Definition: xlogreader.c:161
XLogReaderState * XLogReaderAllocate(int wal_segment_size, const char *waldir, XLogReaderRoutine *routine, void *private_data)
Definition: xlogreader.c:106
XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
Definition: xlogreader.c:1383
void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
Definition: xlogreader.c:231
#define XLogRecGetInfo(decoder)
Definition: xlogreader.h:410
#define XLogRecGetRmid(decoder)
Definition: xlogreader.h:411
#define XLogRecGetData(decoder)
Definition: xlogreader.h:415
#define XL_ROUTINE(...)
Definition: xlogreader.h:117
#define XLogRecMaxBlockId(decoder)
Definition: xlogreader.h:418
#define XLR_INFO_MASK
Definition: xlogrecord.h:62
XLogRecPtr GetXLogReplayRecPtr(TimeLineID *replayTLI)
static XLogReaderState * xlogreader
Definition: xlogrecovery.c:188
void wal_segment_close(XLogReaderState *state)
Definition: xlogutils.c:842
void wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo, TimeLineID *tli_p)
Definition: xlogutils.c:817
void WALReadRaiseError(WALReadError *errinfo)
Definition: xlogutils.c:1020