PostgreSQL Source Code  git master
walsummary.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * walsummary.c
4  * Functions for accessing and managing WAL summary data.
5  *
6  * Portions Copyright (c) 2010-2024, PostgreSQL Global Development Group
7  *
8  * src/backend/backup/walsummary.c
9  *
10  *-------------------------------------------------------------------------
11  */
12 
13 #include "postgres.h"
14 
15 #include <sys/stat.h>
16 #include <unistd.h>
17 
18 #include "access/xlog_internal.h"
19 #include "backup/walsummary.h"
20 #include "common/int.h"
21 #include "utils/wait_event.h"
22 
23 static bool IsWalSummaryFilename(char *filename);
25  const ListCell *b);
26 
27 /*
28  * Get a list of WAL summaries.
29  *
30  * If tli != 0, only WAL summaries with the indicated TLI will be included.
31  *
32  * If start_lsn != InvalidXLogRecPtr, only summaries that end after the
33  * indicated LSN will be included.
34  *
35  * If end_lsn != InvalidXLogRecPtr, only summaries that start before the
36  * indicated LSN will be included.
37  *
38  * The intent is that you can call GetWalSummaries(tli, start_lsn, end_lsn)
39  * to get all WAL summaries on the indicated timeline that overlap the
40  * specified LSN range.
41  */
42 List *
44 {
45  DIR *sdir;
46  struct dirent *dent;
47  List *result = NIL;
48 
49  sdir = AllocateDir(XLOGDIR "/summaries");
50  while ((dent = ReadDir(sdir, XLOGDIR "/summaries")) != NULL)
51  {
52  WalSummaryFile *ws;
53  uint32 tmp[5];
54  TimeLineID file_tli;
55  XLogRecPtr file_start_lsn;
56  XLogRecPtr file_end_lsn;
57 
58  /* Decode filename, or skip if it's not in the expected format. */
59  if (!IsWalSummaryFilename(dent->d_name))
60  continue;
61  sscanf(dent->d_name, "%08X%08X%08X%08X%08X",
62  &tmp[0], &tmp[1], &tmp[2], &tmp[3], &tmp[4]);
63  file_tli = tmp[0];
64  file_start_lsn = ((uint64) tmp[1]) << 32 | tmp[2];
65  file_end_lsn = ((uint64) tmp[3]) << 32 | tmp[4];
66 
67  /* Skip if it doesn't match the filter criteria. */
68  if (tli != 0 && tli != file_tli)
69  continue;
70  if (!XLogRecPtrIsInvalid(start_lsn) && start_lsn >= file_end_lsn)
71  continue;
72  if (!XLogRecPtrIsInvalid(end_lsn) && end_lsn <= file_start_lsn)
73  continue;
74 
75  /* Add it to the list. */
76  ws = palloc(sizeof(WalSummaryFile));
77  ws->tli = file_tli;
78  ws->start_lsn = file_start_lsn;
79  ws->end_lsn = file_end_lsn;
80  result = lappend(result, ws);
81  }
82  FreeDir(sdir);
83 
84  return result;
85 }
86 
87 /*
88  * Build a new list of WAL summaries based on an existing list, but filtering
89  * out summaries that don't match the search parameters.
90  *
91  * If tli != 0, only WAL summaries with the indicated TLI will be included.
92  *
93  * If start_lsn != InvalidXLogRecPtr, only summaries that end after the
94  * indicated LSN will be included.
95  *
96  * If end_lsn != InvalidXLogRecPtr, only summaries that start before the
97  * indicated LSN will be included.
98  */
99 List *
101  XLogRecPtr start_lsn, XLogRecPtr end_lsn)
102 {
103  List *result = NIL;
104  ListCell *lc;
105 
106  /* Loop over input. */
107  foreach(lc, wslist)
108  {
109  WalSummaryFile *ws = lfirst(lc);
110 
111  /* Skip if it doesn't match the filter criteria. */
112  if (tli != 0 && tli != ws->tli)
113  continue;
114  if (!XLogRecPtrIsInvalid(start_lsn) && start_lsn > ws->end_lsn)
115  continue;
116  if (!XLogRecPtrIsInvalid(end_lsn) && end_lsn < ws->start_lsn)
117  continue;
118 
119  /* Add it to the result list. */
120  result = lappend(result, ws);
121  }
122 
123  return result;
124 }
125 
126 /*
127  * Check whether the supplied list of WalSummaryFile objects covers the
128  * whole range of LSNs from start_lsn to end_lsn. This function ignores
129  * timelines, so the caller should probably filter using the appropriate
130  * timeline before calling this.
131  *
132  * If the whole range of LSNs is covered, returns true, otherwise false.
133  * If false is returned, *missing_lsn is set either to InvalidXLogRecPtr
134  * if there are no WAL summary files in the input list, or to the first LSN
135  * in the range that is not covered by a WAL summary file in the input list.
136  */
137 bool
139  XLogRecPtr end_lsn, XLogRecPtr *missing_lsn)
140 {
141  XLogRecPtr current_lsn = start_lsn;
142  ListCell *lc;
143 
144  /* Special case for empty list. */
145  if (wslist == NIL)
146  {
147  *missing_lsn = InvalidXLogRecPtr;
148  return false;
149  }
150 
151  /* Make a private copy of the list and sort it by start LSN. */
152  wslist = list_copy(wslist);
154 
155  /*
156  * Consider summary files in order of increasing start_lsn, advancing the
157  * known-summarized range from start_lsn toward end_lsn.
158  *
159  * Normally, the summary files should cover non-overlapping WAL ranges,
160  * but this algorithm is intended to be correct even in case of overlap.
161  */
162  foreach(lc, wslist)
163  {
164  WalSummaryFile *ws = lfirst(lc);
165 
166  if (ws->start_lsn > current_lsn)
167  {
168  /* We found a gap. */
169  break;
170  }
171  if (ws->end_lsn > current_lsn)
172  {
173  /*
174  * Next summary extends beyond end of previous summary, so extend
175  * the end of the range known to be summarized.
176  */
177  current_lsn = ws->end_lsn;
178 
179  /*
180  * If the range we know to be summarized has reached the required
181  * end LSN, we have proved completeness.
182  */
183  if (current_lsn >= end_lsn)
184  return true;
185  }
186  }
187 
188  /*
189  * We either ran out of summary files without reaching the end LSN, or we
190  * hit a gap in the sequence that resulted in us bailing out of the loop
191  * above.
192  */
193  *missing_lsn = current_lsn;
194  return false;
195 }
196 
197 /*
198  * Open a WAL summary file.
199  *
200  * This will throw an error in case of trouble. As an exception, if
201  * missing_ok = true and the trouble is specifically that the file does
202  * not exist, it will not throw an error and will return a value less than 0.
203  */
204 File
205 OpenWalSummaryFile(WalSummaryFile *ws, bool missing_ok)
206 {
207  char path[MAXPGPATH];
208  File file;
209 
210  snprintf(path, MAXPGPATH,
211  XLOGDIR "/summaries/%08X%08X%08X%08X%08X.summary",
212  ws->tli,
214  LSN_FORMAT_ARGS(ws->end_lsn));
215 
216  file = PathNameOpenFile(path, O_RDONLY);
217  if (file < 0 && (errno != EEXIST || !missing_ok))
218  ereport(ERROR,
220  errmsg("could not open file \"%s\": %m", path)));
221 
222  return file;
223 }
224 
225 /*
226  * Remove a WAL summary file if the last modification time precedes the
227  * cutoff time.
228  */
229 void
231 {
232  char path[MAXPGPATH];
233  struct stat statbuf;
234 
235  snprintf(path, MAXPGPATH,
236  XLOGDIR "/summaries/%08X%08X%08X%08X%08X.summary",
237  ws->tli,
239  LSN_FORMAT_ARGS(ws->end_lsn));
240 
241  if (lstat(path, &statbuf) != 0)
242  {
243  if (errno == ENOENT)
244  return;
245  ereport(ERROR,
247  errmsg("could not stat file \"%s\": %m", path)));
248  }
249  if (statbuf.st_mtime >= cutoff_time)
250  return;
251  if (unlink(path) != 0)
252  ereport(ERROR,
254  errmsg("could not stat file \"%s\": %m", path)));
255  ereport(DEBUG2,
256  (errmsg_internal("removing file \"%s\"", path)));
257 }
258 
259 /*
260  * Test whether a filename looks like a WAL summary file.
261  */
262 static bool
264 {
265  return strspn(filename, "0123456789ABCDEF") == 40 &&
266  strcmp(filename + 40, ".summary") == 0;
267 }
268 
269 /*
270  * Data read callback for use with CreateBlockRefTableReader.
271  */
272 int
273 ReadWalSummary(void *wal_summary_io, void *data, int length)
274 {
275  WalSummaryIO *io = wal_summary_io;
276  int nbytes;
277 
278  nbytes = FileRead(io->file, data, length, io->filepos,
279  WAIT_EVENT_WAL_SUMMARY_READ);
280  if (nbytes < 0)
281  ereport(ERROR,
283  errmsg("could not read file \"%s\": %m",
284  FilePathName(io->file))));
285 
286  io->filepos += nbytes;
287  return nbytes;
288 }
289 
290 /*
291  * Data write callback for use with WriteBlockRefTable.
292  */
293 int
294 WriteWalSummary(void *wal_summary_io, void *data, int length)
295 {
296  WalSummaryIO *io = wal_summary_io;
297  int nbytes;
298 
299  nbytes = FileWrite(io->file, data, length, io->filepos,
300  WAIT_EVENT_WAL_SUMMARY_WRITE);
301  if (nbytes < 0)
302  ereport(ERROR,
304  errmsg("could not write file \"%s\": %m",
305  FilePathName(io->file))));
306  if (nbytes != length)
307  ereport(ERROR,
309  errmsg("could not write file \"%s\": wrote only %d of %d bytes at offset %u",
310  FilePathName(io->file), nbytes,
311  length, (unsigned) io->filepos),
312  errhint("Check free disk space.")));
313 
314  io->filepos += nbytes;
315  return nbytes;
316 }
317 
318 /*
319  * Error-reporting callback for use with CreateBlockRefTableReader.
320  */
321 void
322 ReportWalSummaryError(void *callback_arg, char *fmt,...)
323 {
325  va_list ap;
326  int needed;
327 
329  for (;;)
330  {
331  va_start(ap, fmt);
332  needed = appendStringInfoVA(&buf, fmt, ap);
333  va_end(ap);
334  if (needed == 0)
335  break;
336  enlargeStringInfo(&buf, needed);
337  }
338  ereport(ERROR,
340  errmsg_internal("%s", buf.data));
341 }
342 
343 /*
344  * Comparator to sort a List of WalSummaryFile objects by start_lsn.
345  */
346 static int
348 {
349  WalSummaryFile *ws1 = lfirst(a);
350  WalSummaryFile *ws2 = lfirst(b);
351 
352  return pg_cmp_u64(ws1->start_lsn, ws2->start_lsn);
353 }
unsigned int uint32
Definition: c.h:506
int errmsg_internal(const char *fmt,...)
Definition: elog.c:1159
int errcode_for_file_access(void)
Definition: elog.c:878
int errhint(const char *fmt,...)
Definition: elog.c:1319
int errcode(int sqlerrcode)
Definition: elog.c:855
int errmsg(const char *fmt,...)
Definition: elog.c:1072
#define DEBUG2
Definition: elog.h:29
#define ERROR
Definition: elog.h:39
#define ereport(elevel,...)
Definition: elog.h:149
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2909
int FreeDir(DIR *dir)
Definition: fd.c:2961
File PathNameOpenFile(const char *fileName, int fileFlags)
Definition: fd.c:1575
char * FilePathName(File file)
Definition: fd.c:2461
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2843
static ssize_t FileRead(File file, void *buffer, size_t amount, off_t offset, uint32 wait_event_info)
Definition: fd.h:196
int File
Definition: fd.h:51
static ssize_t FileWrite(File file, const void *buffer, size_t amount, off_t offset, uint32 wait_event_info)
Definition: fd.h:208
static int pg_cmp_u64(uint64 a, uint64 b)
Definition: int.h:501
int b
Definition: isn.c:70
int a
Definition: isn.c:69
static void const char * fmt
va_end(args)
va_start(args, fmt)
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1674
List * lappend(List *list, void *datum)
Definition: list.c:339
List * list_copy(const List *oldlist)
Definition: list.c:1573
void * palloc(Size size)
Definition: mcxt.c:1317
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:41
#define MAXPGPATH
const void * data
static char * filename
Definition: pg_dumpall.c:119
#define lfirst(lc)
Definition: pg_list.h:172
#define NIL
Definition: pg_list.h:68
static char * buf
Definition: pg_test_fsync.c:73
#define snprintf
Definition: port.h:238
int appendStringInfoVA(StringInfo str, const char *fmt, va_list args)
Definition: stringinfo.c:139
void enlargeStringInfo(StringInfo str, int needed)
Definition: stringinfo.c:289
void initStringInfo(StringInfo str)
Definition: stringinfo.c:59
Definition: dirent.c:26
Definition: pg_list.h:54
XLogRecPtr end_lsn
Definition: walsummary.h:30
TimeLineID tli
Definition: walsummary.h:31
XLogRecPtr start_lsn
Definition: walsummary.h:29
off_t filepos
Definition: walsummary.h:24
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
__time64_t st_mtime
Definition: win32_port.h:275
void RemoveWalSummaryIfOlderThan(WalSummaryFile *ws, time_t cutoff_time)
Definition: walsummary.c:230
static bool IsWalSummaryFilename(char *filename)
Definition: walsummary.c:263
File OpenWalSummaryFile(WalSummaryFile *ws, bool missing_ok)
Definition: walsummary.c:205
static int ListComparatorForWalSummaryFiles(const ListCell *a, const ListCell *b)
Definition: walsummary.c:347
List * GetWalSummaries(TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
Definition: walsummary.c:43
bool WalSummariesAreComplete(List *wslist, XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogRecPtr *missing_lsn)
Definition: walsummary.c:138
int ReadWalSummary(void *wal_summary_io, void *data, int length)
Definition: walsummary.c:273
List * FilterWalSummaries(List *wslist, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
Definition: walsummary.c:100
void ReportWalSummaryError(void *callback_arg, char *fmt,...)
Definition: walsummary.c:322
int WriteWalSummary(void *wal_summary_io, void *data, int length)
Definition: walsummary.c:294
#define lstat(path, sb)
Definition: win32_port.h:285
#define XLOGDIR
#define LSN_FORMAT_ARGS(lsn)
Definition: xlogdefs.h:43
#define XLogRecPtrIsInvalid(r)
Definition: xlogdefs.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28
uint32 TimeLineID
Definition: xlogdefs.h:59